1use crate::matrix::Matrix;
48use crate::network::NetworkError;
49use crate::param::{Linear, Param};
50#[cfg(feature = "cuda")]
51use cudarc::driver::PushKernelArg;
52use rand::rngs::StdRng;
53use rayon::prelude::*;
54
55#[derive(Clone, Debug, PartialEq)]
58pub struct FeatureMap {
59 pub channels: usize,
60 pub height: usize,
61 pub width: usize,
62 pub data: Vec<f32>,
63}
64
65impl FeatureMap {
66 pub fn new(channels: usize, height: usize, width: usize) -> Self {
68 Self {
69 channels,
70 height,
71 width,
72 data: vec![0.0; channels * height * width],
73 }
74 }
75
76 pub fn from_vec(
78 channels: usize,
79 height: usize,
80 width: usize,
81 data: Vec<f32>,
82 ) -> Result<Self, NetworkError> {
83 if data.len() != channels * height * width {
84 return Err(NetworkError::InvalidTarget {
85 expected: channels * height * width,
86 actual: data.len(),
87 });
88 }
89 Ok(Self {
90 channels,
91 height,
92 width,
93 data,
94 })
95 }
96
97 pub fn pixels(&self) -> usize {
99 self.height * self.width
100 }
101
102 pub fn plane(&self, channel: usize) -> &[f32] {
104 let pixels = self.pixels();
105 &self.data[channel * pixels..(channel + 1) * pixels]
106 }
107
108 pub fn to_tokens(&self) -> Matrix {
111 let pixels = self.pixels();
112 let mut tokens = Matrix::new(pixels, self.channels);
113 for channel in 0..self.channels {
114 let plane = self.plane(channel);
115 for (pixel, value) in plane.iter().enumerate() {
116 tokens.data[pixel * self.channels + channel] = *value;
117 }
118 }
119 tokens
120 }
121
122 pub fn from_tokens(tokens: &Matrix, height: usize, width: usize) -> Result<Self, NetworkError> {
124 if tokens.rows != height * width {
125 return Err(NetworkError::InvalidTarget {
126 expected: height * width,
127 actual: tokens.rows,
128 });
129 }
130 let mut map = Self::new(tokens.cols, height, width);
131 for channel in 0..tokens.cols {
132 for pixel in 0..tokens.rows {
133 map.data[channel * tokens.rows + pixel] =
134 tokens.data[pixel * tokens.cols + channel];
135 }
136 }
137 Ok(map)
138 }
139}
140
141#[derive(Clone, Debug)]
148pub struct Dense {
149 pub weight: Matrix,
150 pub bias: Option<Vec<f32>>,
151 quantized: Option<crate::quantized::Quantized>,
154}
155
156impl Dense {
157 pub fn new(weight: Matrix, bias: Option<Vec<f32>>) -> Result<Self, NetworkError> {
159 if bias.as_ref().is_some_and(|bias| bias.len() != weight.rows) {
160 return Err(NetworkError::InvalidTarget {
161 expected: weight.rows,
162 actual: bias.as_ref().map_or(0, |bias| bias.len()),
163 });
164 }
165 Ok(Self {
166 weight,
167 bias,
168 quantized: None,
169 })
170 }
171
172 pub fn out_dim(&self) -> usize {
174 self.weight.rows
175 }
176
177 pub fn in_dim(&self) -> usize {
179 self.weight.cols
180 }
181
182 pub fn quantize(&mut self) {
188 if self.quantized.is_none() {
189 self.quantized = Some(crate::quantized::Quantized::from_matrix(&self.weight));
190 self.weight.data = Vec::new();
191 }
192 }
193
194 pub(crate) fn weight_f32(&self) -> std::borrow::Cow<'_, Matrix> {
198 match &self.quantized {
199 Some(quantized) => std::borrow::Cow::Owned(quantized.dequantize()),
200 None => std::borrow::Cow::Borrowed(&self.weight),
201 }
202 }
203
204 pub fn is_quantized(&self) -> bool {
206 self.quantized.is_some()
207 }
208
209 pub fn forward(&self, tokens: &Matrix) -> Result<Matrix, NetworkError> {
211 if tokens.cols != self.weight.cols {
212 return Err(NetworkError::InvalidTarget {
213 expected: self.weight.cols,
214 actual: tokens.cols,
215 });
216 }
217 let mut output = match &self.quantized {
218 Some(weight) => weight.matmul_rhs_transposed(tokens),
219 None => {
220 let mut output = Matrix::new(tokens.rows, self.weight.rows);
221 tokens.dot_rhs_transposed(&self.weight, &mut output);
222 output
223 }
224 };
225 if let Some(bias) = &self.bias {
226 output
227 .data
228 .par_chunks_mut(self.weight.rows)
229 .for_each(|row| {
230 for (value, bias) in row.iter_mut().zip(bias) {
231 *value += bias;
232 }
233 });
234 }
235 Ok(output)
236 }
237
238 pub fn apply(&self, input: &[f32]) -> Result<Vec<f32>, NetworkError> {
240 let row = Matrix::from_vec(1, input.len(), input.to_vec());
241 Ok(self.forward(&row)?.data)
242 }
243}
244
245#[derive(Clone, Debug)]
247pub struct Conv2d {
248 pub weight: Matrix,
251 pub bias: Option<Vec<f32>>,
252 pub in_channels: usize,
253 pub kernel: usize,
254 pub stride: usize,
255 pub padding: usize,
256}
257
258impl Conv2d {
259 pub fn new(
265 weight: Matrix,
266 bias: Option<Vec<f32>>,
267 in_channels: usize,
268 kernel: usize,
269 stride: usize,
270 padding: usize,
271 ) -> Result<Self, NetworkError> {
272 if kernel == 0 || stride == 0 {
273 return Err(NetworkError::InvalidConfig(
274 "a convolution needs a kernel and a stride of at least one".into(),
275 ));
276 }
277 if weight.cols != in_channels * kernel * kernel {
278 return Err(NetworkError::InvalidConfig(format!(
279 "a {kernel}x{kernel} convolution over {in_channels} channels needs {} weights per \
280 output channel, and this one has {}",
281 in_channels * kernel * kernel,
282 weight.cols
283 )));
284 }
285 if bias.as_ref().is_some_and(|bias| bias.len() != weight.rows) {
286 return Err(NetworkError::InvalidConfig(
287 "a convolution's bias has one value per output channel".into(),
288 ));
289 }
290 Ok(Self {
291 weight,
292 bias,
293 in_channels,
294 kernel,
295 stride,
296 padding,
297 })
298 }
299
300 pub fn out_channels(&self) -> usize {
302 self.weight.rows
303 }
304
305 pub fn output_size(&self, height: usize, width: usize) -> (usize, usize) {
307 let size = |length: usize| match (length + 2 * self.padding).checked_sub(self.kernel) {
310 Some(span) => span / self.stride + 1,
311 None => 0,
312 };
313 (size(height), size(width))
314 }
315
316 pub fn forward(&self, input: &FeatureMap) -> Result<FeatureMap, NetworkError> {
318 if input.channels != self.in_channels {
319 return Err(NetworkError::InvalidConfig(format!(
320 "this convolution reads {} channels and was handed {}",
321 self.in_channels, input.channels
322 )));
323 }
324 let (out_height, out_width) = self.output_size(input.height, input.width);
325 if out_height == 0 || out_width == 0 {
326 return Err(NetworkError::InvalidConfig(format!(
327 "a {}x{} kernel over a {}x{} input leaves nothing",
328 self.kernel, self.kernel, input.height, input.width
329 )));
330 }
331
332 let columns = self.im2col(input, out_height, out_width);
333 let mut product = Matrix::new(out_height * out_width, self.out_channels());
334 columns.dot_rhs_transposed(&self.weight, &mut product);
335
336 let mut output = FeatureMap::new(self.out_channels(), out_height, out_width);
338 let pixels = out_height * out_width;
339 output
340 .data
341 .par_chunks_mut(pixels)
342 .enumerate()
343 .for_each(|(channel, plane)| {
344 let bias = self.bias.as_ref().map_or(0.0, |bias| bias[channel]);
345 for (pixel, value) in plane.iter_mut().enumerate() {
346 *value = product.data[pixel * self.out_channels() + channel] + bias;
347 }
348 });
349 Ok(output)
350 }
351
352 fn im2col(&self, input: &FeatureMap, out_height: usize, out_width: usize) -> Matrix {
355 let patch = self.in_channels * self.kernel * self.kernel;
356 let mut columns = Matrix::new(out_height * out_width, patch);
357 columns
358 .data
359 .par_chunks_mut(patch)
360 .enumerate()
361 .for_each(|(pixel, row)| {
362 let (out_y, out_x) = (pixel / out_width, pixel % out_width);
363 let top = (out_y * self.stride) as isize - self.padding as isize;
364 let left = (out_x * self.stride) as isize - self.padding as isize;
365 for channel in 0..self.in_channels {
366 let plane = channel * input.height * input.width;
367 for row_offset in 0..self.kernel {
368 let y = top + row_offset as isize;
369 if y < 0 || y >= input.height as isize {
370 continue;
371 }
372 let source = plane + y as usize * input.width;
373 let target = (channel * self.kernel + row_offset) * self.kernel;
374 for column_offset in 0..self.kernel {
375 let x = left + column_offset as isize;
376 if x >= 0 && x < input.width as isize {
377 row[target + column_offset] = input.data[source + x as usize];
378 }
379 }
380 }
381 }
382 });
383 columns
384 }
385}
386
387#[derive(Clone, Debug, PartialEq)]
394pub struct ImageBatch {
395 pub batch: usize,
396 pub height: usize,
397 pub width: usize,
398 pub tokens: Matrix,
399}
400
401impl ImageBatch {
402 pub fn new(batch: usize, height: usize, width: usize, channels: usize) -> Self {
403 Self {
404 batch,
405 height,
406 width,
407 tokens: Matrix::new(batch * height * width, channels),
408 }
409 }
410
411 pub fn from_tokens(batch: usize, height: usize, width: usize, tokens: Matrix) -> Self {
413 debug_assert_eq!(tokens.rows, batch * height * width);
414 Self {
415 batch,
416 height,
417 width,
418 tokens,
419 }
420 }
421
422 pub fn channels(&self) -> usize {
423 self.tokens.cols
424 }
425
426 pub fn pixels(&self) -> usize {
428 self.height * self.width
429 }
430
431 pub fn like(&self, channels: usize) -> Self {
433 Self::new(self.batch, self.height, self.width, channels)
434 }
435
436 pub fn pixel(&self, sample: usize, y: usize, x: usize) -> &[f32] {
438 self.tokens.row((sample * self.height + y) * self.width + x)
439 }
440
441 pub fn pixel_mut(&mut self, sample: usize, y: usize, x: usize) -> &mut [f32] {
442 let row = (sample * self.height + y) * self.width + x;
443 self.tokens.row_mut(row)
444 }
445
446 pub fn add(&mut self, other: &ImageBatch) {
448 debug_assert_eq!(self.tokens.data.len(), other.tokens.data.len());
449 for (slot, value) in self.tokens.data.iter_mut().zip(&other.tokens.data) {
450 *slot += value;
451 }
452 }
453}
454
455#[cfg(feature = "cuda")]
459pub struct DeviceImageBatch {
460 pub batch: usize,
461 pub height: usize,
462 pub width: usize,
463 channels: usize,
464 pub(crate) data: cudarc::driver::CudaSlice<f32>,
465 context: std::sync::Arc<crate::gpu_transformer::GpuContext>,
466}
467
468#[cfg(feature = "cuda")]
469impl DeviceImageBatch {
470 pub(crate) fn upload(
471 context: &std::sync::Arc<crate::gpu_transformer::GpuContext>,
472 batch: &ImageBatch,
473 ) -> Result<Self, NetworkError> {
474 Ok(Self {
475 batch: batch.batch,
476 height: batch.height,
477 width: batch.width,
478 channels: batch.channels(),
479 data: context.upload(&batch.tokens)?,
480 context: context.clone(),
481 })
482 }
483
484 pub fn download(&self) -> Result<ImageBatch, NetworkError> {
486 let mut tokens = Matrix::new(self.batch * self.pixels(), self.channels);
487 self.context.download(&self.data, &mut tokens)?;
488 Ok(ImageBatch::from_tokens(
489 self.batch,
490 self.height,
491 self.width,
492 tokens,
493 ))
494 }
495
496 pub fn channels(&self) -> usize {
497 self.channels
498 }
499
500 pub fn pixels(&self) -> usize {
501 self.height * self.width
502 }
503
504 pub fn like(&self, channels: usize) -> Result<Self, NetworkError> {
506 let data = self.context.zeros(self.batch * self.pixels() * channels)?;
507 Ok(Self {
508 channels,
509 ..self.with_data(data)
510 })
511 }
512
513 fn with_data(&self, data: cudarc::driver::CudaSlice<f32>) -> Self {
515 Self {
516 data,
517 context: self.context.clone(),
518 ..*self
519 }
520 }
521}
522
523#[cfg(feature = "cuda")]
524impl Clone for DeviceImageBatch {
525 fn clone(&self) -> Self {
526 self.with_data(self.data.clone())
527 }
528}
529
530#[cfg(feature = "cuda")]
531impl crate::param::CudaDevice {
532 pub fn upload(&self, batch: &ImageBatch) -> Result<DeviceImageBatch, NetworkError> {
534 DeviceImageBatch::upload(self.context(), batch)
535 }
536
537 pub fn download(&self, batch: &DeviceImageBatch) -> Result<ImageBatch, NetworkError> {
539 batch.download()
540 }
541}
542
543#[derive(Clone, Copy, Debug, PartialEq, Eq)]
546pub struct ConvGeometry {
547 pub batch: usize,
548 pub channels: usize,
549 pub height: usize,
550 pub width: usize,
551 pub kernel: usize,
552 pub stride: usize,
553 pub padding: usize,
554 pub out_height: usize,
555 pub out_width: usize,
556}
557
558impl ConvGeometry {
559 pub fn is_pointwise(&self) -> bool {
562 self.kernel == 1 && self.stride == 1 && self.padding == 0
563 }
564
565 pub fn input_elements(&self) -> usize {
567 self.batch * self.height * self.width * self.channels
568 }
569
570 pub fn column_elements(&self) -> usize {
573 self.batch * self.out_height * self.out_width * self.patch()
574 }
575
576 pub fn patch(&self) -> usize {
578 self.channels * self.kernel * self.kernel
579 }
580
581 pub fn rows(&self) -> usize {
584 self.batch * self.out_height * self.out_width
585 }
586}
587
588#[derive(Clone, Debug)]
595pub struct TrainableConv2d {
596 pub weight: Linear,
598 pub bias: Param,
601 has_bias: bool,
602 in_channels: usize,
603 kernel: usize,
604 stride: usize,
605 padding: usize,
606 recompute_columns: bool,
609}
610
611pub struct ConvCache {
619 columns: Columns,
620 shape: ConvGeometry,
621}
622
623impl ConvCache {
624 pub fn holds_columns(&self) -> bool {
628 match &self.columns {
629 Columns::Input(_) => false,
630 #[cfg(feature = "cuda")]
631 Columns::DeviceInput(_) | Columns::DeviceElsewhere => false,
632 _ => true,
633 }
634 }
635}
636
637enum Columns {
638 Host(Matrix),
639 #[cfg(feature = "cuda")]
640 Device(cudarc::driver::CudaSlice<f32>),
641 Input(ImageBatch),
643 #[cfg(feature = "cuda")]
646 DeviceInput(cudarc::driver::CudaSlice<f32>),
647 #[cfg(feature = "cuda")]
649 DeviceNarrow(cudarc::driver::CudaSlice<u8>),
650 #[cfg(feature = "cuda")]
653 DeviceElsewhere,
654}
655
656impl std::fmt::Debug for ConvCache {
657 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
658 f.debug_struct("ConvCache")
659 .field("shape", &self.shape)
660 .finish()
661 }
662}
663
664impl TrainableConv2d {
665 pub fn new(
668 in_channels: usize,
669 out_channels: usize,
670 kernel: usize,
671 stride: usize,
672 rng: &mut StdRng,
673 ) -> Result<Self, NetworkError> {
674 if kernel == 0 || stride == 0 || in_channels == 0 || out_channels == 0 {
675 return Err(NetworkError::InvalidConfig(
676 "a convolution needs a non-zero kernel, stride and channel count".into(),
677 ));
678 }
679 let fan_in = in_channels * kernel * kernel;
680 Ok(Self {
681 weight: Linear {
682 weight: Param::he_uniform(out_channels, fan_in, fan_in, rng),
683 lora: None,
684 },
685 bias: Param::zeros(1, out_channels),
686 has_bias: true,
687 in_channels,
688 kernel,
689 stride,
690 padding: kernel / 2,
691 recompute_columns: false,
692 })
693 }
694
695 pub fn zeroed(
699 in_channels: usize,
700 out_channels: usize,
701 kernel: usize,
702 rng: &mut StdRng,
703 ) -> Result<Self, NetworkError> {
704 let mut conv = Self::new(in_channels, out_channels, kernel, 1, rng)?;
705 conv.weight.weight = Param::zeros(out_channels, in_channels * kernel * kernel);
706 Ok(conv)
707 }
708
709 pub fn without_bias(mut self) -> Self {
713 self.has_bias = false;
714 self.bias = Param::zeros(1, self.bias.value.cols);
715 self
716 }
717
718 pub fn has_bias(&self) -> bool {
719 self.has_bias
720 }
721
722 pub fn out_channels(&self) -> usize {
723 self.bias.value.cols
724 }
725
726 pub fn in_channels(&self) -> usize {
727 self.in_channels
728 }
729
730 pub fn output_size(&self, height: usize, width: usize) -> (usize, usize) {
731 let size = |length: usize| {
732 (length + 2 * self.padding).saturating_sub(self.kernel) / self.stride + 1
733 };
734 (size(height), size(width))
735 }
736
737 pub fn geometry(&self, batch: usize, height: usize, width: usize) -> ConvGeometry {
739 let (out_height, out_width) = self.output_size(height, width);
740 ConvGeometry {
741 batch,
742 channels: self.in_channels,
743 height,
744 width,
745 kernel: self.kernel,
746 stride: self.stride,
747 padding: self.padding,
748 out_height,
749 out_width,
750 }
751 }
752
753 pub fn set_recompute_columns(&mut self, recompute: bool) {
780 self.recompute_columns = recompute;
781 }
782
783 pub fn recomputes_columns(&self) -> bool {
785 self.recompute_columns
786 }
787
788 pub fn to_cuda_on(&mut self, device: &crate::param::CudaDevice) -> Result<(), NetworkError> {
794 self.weight.to_cuda_on(device)?;
795 if self.has_bias {
796 self.bias.to_cuda_on(device)?;
797 }
798 Ok(())
799 }
800
801 pub fn to_cpu(&mut self) -> Result<(), NetworkError> {
803 self.weight.to_cpu()?;
804 self.bias.to_cpu()
805 }
806
807 pub fn forward(&self, input: &ImageBatch) -> Result<(ImageBatch, ConvCache), NetworkError> {
808 if input.channels() != self.in_channels {
809 return Err(NetworkError::InvalidConfig(format!(
810 "this convolution reads {} channels and was given {}",
811 self.in_channels,
812 input.channels()
813 )));
814 }
815 let shape = self.geometry(input.batch, input.height, input.width);
816 #[cfg(feature = "cuda")]
817 if let Some(device) = &self.weight.weight.device {
818 return self.forward_on_device(input, device);
819 }
820 let columns = self.im2col(input, &shape);
821 let mut tokens = self.weight.forward(&columns);
822 self.add_bias(&mut tokens);
823 Ok((
824 ImageBatch::from_tokens(input.batch, shape.out_height, shape.out_width, tokens),
825 ConvCache {
826 columns: match self.recompute_columns {
827 true => Columns::Input(input.clone()),
828 false => Columns::Host(columns),
829 },
830 shape,
831 },
832 ))
833 }
834
835 pub fn backward(
837 &mut self,
838 cache: &ConvCache,
839 grad_output: &ImageBatch,
840 ) -> Result<ImageBatch, NetworkError> {
841 match &cache.columns {
842 Columns::Host(columns) => {
843 self.accumulate_bias_grad(&grad_output.tokens);
844 let grad_columns = self.weight.backward(columns, &grad_output.tokens);
845 Ok(self.col2im(&grad_columns, &cache.shape))
846 }
847 #[cfg(feature = "cuda")]
848 Columns::Device(_)
849 | Columns::DeviceInput(_)
850 | Columns::DeviceNarrow(_)
851 | Columns::DeviceElsewhere => self.backward_on_device(cache, grad_output),
852 Columns::Input(input) => {
853 self.accumulate_bias_grad(&grad_output.tokens);
854 let columns = self.im2col(input, &cache.shape);
855 let grad_columns = self.weight.backward(&columns, &grad_output.tokens);
856 Ok(self.col2im(&grad_columns, &cache.shape))
857 }
858 }
859 }
860
861 #[cfg(feature = "cuda")]
865 pub fn infer_device(
866 &self,
867 input: &DeviceImageBatch,
868 silu: bool,
869 ) -> Result<DeviceImageBatch, NetworkError> {
870 self.infer_device_with(input, silu, None)
871 }
872
873 #[cfg(feature = "cuda")]
878 pub(crate) fn infer_device_with(
879 &self,
880 input: &DeviceImageBatch,
881 silu: bool,
882 narrow_weight: Option<&cudarc::driver::CudaSlice<u8>>,
883 ) -> Result<DeviceImageBatch, NetworkError> {
884 if input.channels != self.in_channels {
885 return Err(NetworkError::InvalidConfig(format!(
886 "this convolution reads {} channels and was given {}",
887 self.in_channels, input.channels
888 )));
889 }
890 let device = self.weight.weight.device.as_ref().ok_or_else(|| {
891 NetworkError::InvalidConfig("infer_device needs the weight on a device".into())
892 })?;
893 let shape = self.geometry(input.batch, input.height, input.width);
894 let context = device.context();
895 let mut product = match ((self.kernel, self.stride, self.padding), narrow_weight) {
896 ((1, 1, 0), _) => device.matmul_rhs_transposed_device(&input.data, shape.rows())?,
897 (_, Some(weight)) => device.matmul_rhs_transposed_bf16(
898 &context.im2col_bf16(&input.data, &shape)?,
899 weight,
900 shape.rows(),
901 )?,
902 _ => device.matmul_rhs_transposed_device(
903 &context.im2col(&input.data, &shape)?,
904 shape.rows(),
905 )?,
906 };
907 let bias = self
908 .bias
909 .device
910 .as_ref()
911 .filter(|_| self.has_bias)
912 .ok_or_else(|| {
913 NetworkError::InvalidConfig("infer_device needs a bias on the device".into())
914 })?;
915 context.add_bias_rows_then(&mut product, bias.value(), self.out_channels(), silu)?;
916 Ok(DeviceImageBatch {
917 batch: input.batch,
918 height: shape.out_height,
919 width: shape.out_width,
920 channels: self.out_channels(),
921 data: product,
922 context: context.clone(),
923 })
924 }
925
926 #[cfg(feature = "cuda")]
930 pub(crate) fn bf16_weight(
931 &self,
932 ) -> Result<Option<cudarc::driver::CudaSlice<u8>>, NetworkError> {
933 let Some(device) = self.weight.weight.device.as_ref() else {
934 return Ok(None);
935 };
936 let context = device.context();
937 match context.mixed_precision && (self.kernel, self.stride, self.padding) != (1, 1, 0) {
938 true => context.bf16_bytes(device.value()).map(Some),
939 false => Ok(None),
940 }
941 }
942
943 #[cfg(feature = "cuda")]
946 pub fn forward_device(
947 &self,
948 input: &DeviceImageBatch,
949 ) -> Result<(DeviceImageBatch, ConvCache), NetworkError> {
950 self.forward_device_keeping(input, true)
951 }
952
953 #[cfg(feature = "cuda")]
958 pub(crate) fn forward_device_keeping(
959 &self,
960 input: &DeviceImageBatch,
961 keep_input: bool,
962 ) -> Result<(DeviceImageBatch, ConvCache), NetworkError> {
963 if input.channels != self.in_channels {
964 return Err(NetworkError::InvalidConfig(format!(
965 "this convolution reads {} channels and was given {}",
966 self.in_channels, input.channels
967 )));
968 }
969 let device = self.weight.weight.device.as_ref().ok_or_else(|| {
970 NetworkError::InvalidConfig("forward_device needs the weight on a device".into())
971 })?;
972 let shape = self.geometry(input.batch, input.height, input.width);
973 let context = device.context();
974 let (mut product, columns) = match (shape.is_pointwise(), context.mixed_precision) {
977 (true, _) => (
978 device.matmul_rhs_transposed_device(&input.data, shape.rows())?,
979 None,
980 ),
981 (false, true) => {
982 let columns = context.im2col_bf16(&input.data, &shape)?;
983 let weight = context.bf16_bytes(device.value())?;
984 (
985 device.matmul_rhs_transposed_bf16(&columns, &weight, shape.rows())?,
986 Some(Columns::DeviceNarrow(columns)),
987 )
988 }
989 (false, false) => {
990 let columns = context.im2col(&input.data, &shape)?;
991 (
992 device.matmul_rhs_transposed_device(&columns, shape.rows())?,
993 Some(Columns::Device(columns)),
994 )
995 }
996 };
997 if let Some(bias) = self.bias.device.as_ref().filter(|_| self.has_bias) {
998 context.add_bias_rows(&mut product, bias.value(), self.out_channels())?;
999 }
1000 Ok((
1001 DeviceImageBatch {
1002 batch: input.batch,
1003 height: shape.out_height,
1004 width: shape.out_width,
1005 channels: self.out_channels(),
1006 data: product,
1007 context: context.clone(),
1008 },
1009 ConvCache {
1010 columns: match columns.filter(|_| !self.recompute_columns) {
1015 Some(columns) => columns,
1016 None if keep_input => Columns::DeviceInput(input.data.clone()),
1017 None => Columns::DeviceElsewhere,
1018 },
1019 shape,
1020 },
1021 ))
1022 }
1023
1024 #[cfg(feature = "cuda")]
1029 pub fn backward_device(
1030 &mut self,
1031 cache: &ConvCache,
1032 grad_output: &DeviceImageBatch,
1033 ) -> Result<DeviceImageBatch, NetworkError> {
1034 self.backward_device_from(cache, None, grad_output)
1035 }
1036
1037 #[cfg(feature = "cuda")]
1041 pub(crate) fn backward_device_from(
1042 &mut self,
1043 cache: &ConvCache,
1044 input: Option<&DeviceImageBatch>,
1045 grad_output: &DeviceImageBatch,
1046 ) -> Result<DeviceImageBatch, NetworkError> {
1047 let shape = &cache.shape;
1048 let input = match &cache.columns {
1049 Columns::DeviceInput(input) => Some(input),
1050 Columns::DeviceElsewhere => input.map(|batch| &batch.data),
1051 _ => None,
1052 };
1053 let device = self.weight.weight.device.as_mut().ok_or_else(|| {
1054 NetworkError::InvalidConfig("backward_device needs the weight on a device".into())
1055 })?;
1056 let context = device.context().clone();
1057 if context.mixed_precision && !shape.is_pointwise() {
1058 let rebuilt;
1059 let columns = match (&cache.columns, input) {
1060 (Columns::DeviceNarrow(columns), _) => columns,
1061 (_, Some(input)) => {
1062 rebuilt = context.im2col_bf16(input, shape)?;
1063 &rebuilt
1064 }
1065 _ => {
1066 return Err(NetworkError::InvalidConfig(
1067 "backward_device needs a cache from a mixed-precision forward pass".into(),
1068 ));
1069 }
1070 };
1071 let grad = context.bf16_bytes(&grad_output.data)?;
1072 device.accumulate_grad_bf16(&grad, columns, shape.rows())?;
1073 let weight = context.bf16_bytes(device.value())?;
1074 let grad_columns = device.matmul_bf16(&grad, &weight, shape.rows())?;
1075 self.accumulate_bias_device(&context, grad_output, shape.rows())?;
1076 return Ok(DeviceImageBatch {
1077 batch: shape.batch,
1078 height: shape.height,
1079 width: shape.width,
1080 channels: shape.channels,
1081 data: context.col2im_bf16(&grad_columns, shape)?,
1082 context,
1083 });
1084 }
1085 let rebuilt;
1086 let columns = match (&cache.columns, input) {
1087 (Columns::Device(columns), _) => columns,
1088 (_, Some(input)) if shape.is_pointwise() => input,
1089 (_, Some(input)) => {
1090 rebuilt = context.im2col(input, shape)?;
1091 &rebuilt
1092 }
1093 _ => {
1094 return Err(NetworkError::InvalidConfig(
1095 "backward_device needs a cache from a device forward pass".into(),
1096 ));
1097 }
1098 };
1099 device.accumulate_grad_device(&grad_output.data, columns, shape.rows())?;
1100 let grad_columns = device.matmul_device(&grad_output.data, shape.rows())?;
1101 self.accumulate_bias_device(&context, grad_output, shape.rows())?;
1102 Ok(DeviceImageBatch {
1103 batch: shape.batch,
1104 height: shape.height,
1105 width: shape.width,
1106 channels: shape.channels,
1107 data: match shape.is_pointwise() {
1108 true => grad_columns,
1109 false => context.col2im(&grad_columns, shape)?,
1110 },
1111 context,
1112 })
1113 }
1114
1115 #[cfg(feature = "cuda")]
1116 fn accumulate_bias_device(
1117 &mut self,
1118 context: &std::sync::Arc<crate::gpu_transformer::GpuContext>,
1119 grad_output: &DeviceImageBatch,
1120 rows: usize,
1121 ) -> Result<(), NetworkError> {
1122 if let Some(bias) = self.bias.device.as_mut().filter(|_| self.has_bias) {
1123 let ones = context.filled(rows, 1.0)?;
1126 bias.accumulate_grad_device(&ones, &grad_output.data, rows)?;
1127 }
1128 Ok(())
1129 }
1130
1131 #[cfg(feature = "cuda")]
1133 fn forward_on_device(
1134 &self,
1135 input: &ImageBatch,
1136 device: &crate::gpu_transformer::DeviceParam,
1137 ) -> Result<(ImageBatch, ConvCache), NetworkError> {
1138 let resident = DeviceImageBatch::upload(device.context(), input)?;
1139 let (output, cache) = self.forward_device(&resident)?;
1140 Ok((output.download()?, cache))
1141 }
1142
1143 #[cfg(feature = "cuda")]
1144 fn backward_on_device(
1145 &mut self,
1146 cache: &ConvCache,
1147 grad_output: &ImageBatch,
1148 ) -> Result<ImageBatch, NetworkError> {
1149 let context = self
1150 .weight
1151 .weight
1152 .device
1153 .as_ref()
1154 .expect("a device-resident cache means a device-resident weight")
1155 .context()
1156 .clone();
1157 let resident = DeviceImageBatch::upload(&context, grad_output)?;
1158 self.backward_device(cache, &resident)?.download()
1159 }
1160
1161 fn add_bias(&self, tokens: &mut Matrix) {
1162 if !self.has_bias {
1163 return;
1164 }
1165 let bias = &self.bias.value.data;
1166 tokens.data.par_chunks_mut(tokens.cols).for_each(|row| {
1167 for (value, bias) in row.iter_mut().zip(bias) {
1168 *value += bias;
1169 }
1170 });
1171 }
1172
1173 fn accumulate_bias_grad(&mut self, grad_output: &Matrix) {
1178 if self.bias.is_frozen() || !self.has_bias {
1179 return;
1180 }
1181 for row in 0..grad_output.rows {
1182 for (slot, value) in self.bias.grad.data.iter_mut().zip(grad_output.row(row)) {
1183 *slot += value;
1184 }
1185 }
1186 }
1187
1188 pub fn params_mut(&mut self) -> Vec<&mut Param> {
1189 let mut params = self.weight.params_mut();
1190 if self.has_bias {
1191 params.push(&mut self.bias);
1192 }
1193 params
1194 }
1195
1196 fn im2col(&self, input: &ImageBatch, shape: &ConvGeometry) -> Matrix {
1199 let patch = shape.patch();
1200 let mut columns = Matrix::new(shape.rows(), patch);
1201 let (kernel, stride, padding) = (self.kernel, self.stride, self.padding);
1202 let channels = self.in_channels;
1203 columns
1204 .data
1205 .par_chunks_mut(patch)
1206 .enumerate()
1207 .for_each(|(index, row)| {
1208 let plane = shape.out_height * shape.out_width;
1209 let (sample, rest) = (index / plane, index % plane);
1210 let (out_y, out_x) = (rest / shape.out_width, rest % shape.out_width);
1211 for ky in 0..kernel {
1212 let y = (out_y * stride + ky) as isize - padding as isize;
1213 if y < 0 || y as usize >= input.height {
1214 continue;
1215 }
1216 for kx in 0..kernel {
1217 let x = (out_x * stride + kx) as isize - padding as isize;
1218 if x < 0 || x as usize >= input.width {
1219 continue;
1220 }
1221 let source = input.pixel(sample, y as usize, x as usize);
1222 let offset = (ky * kernel + kx) * channels;
1223 row[offset..offset + channels].copy_from_slice(source);
1224 }
1225 }
1226 });
1227 columns
1228 }
1229
1230 fn col2im(&self, columns: &Matrix, shape: &ConvGeometry) -> ImageBatch {
1233 let mut grad = ImageBatch::new(shape.batch, shape.height, shape.width, self.in_channels);
1234 for sample in 0..shape.batch {
1235 for out_y in 0..shape.out_height {
1236 for out_x in 0..shape.out_width {
1237 let row =
1238 columns.row((sample * shape.out_height + out_y) * shape.out_width + out_x);
1239 for ky in 0..self.kernel {
1240 let y = (out_y * self.stride + ky) as isize - self.padding as isize;
1241 if y < 0 || y as usize >= shape.height {
1242 continue;
1243 }
1244 for kx in 0..self.kernel {
1245 let x = (out_x * self.stride + kx) as isize - self.padding as isize;
1246 if x < 0 || x as usize >= shape.width {
1247 continue;
1248 }
1249 let offset = (ky * self.kernel + kx) * self.in_channels;
1250 let target = grad.pixel_mut(sample, y as usize, x as usize);
1251 for channel in 0..self.in_channels {
1252 target[channel] += row[offset + channel];
1253 }
1254 }
1255 }
1256 }
1257 }
1258 }
1259 grad
1260 }
1261}
1262
1263#[derive(Clone, Debug)]
1270pub struct GroupNorm {
1271 pub groups: usize,
1272 pub weight: Vec<f32>,
1273 pub bias: Vec<f32>,
1274 pub eps: f32,
1275}
1276
1277impl GroupNorm {
1278 pub fn new(
1280 groups: usize,
1281 weight: Vec<f32>,
1282 bias: Vec<f32>,
1283 eps: f32,
1284 ) -> Result<Self, NetworkError> {
1285 if groups == 0 || weight.is_empty() || weight.len() % groups != 0 {
1286 return Err(NetworkError::InvalidConfig(format!(
1287 "{} channels do not divide into {groups} groups",
1288 weight.len()
1289 )));
1290 }
1291 if bias.len() != weight.len() {
1292 return Err(NetworkError::InvalidTarget {
1293 expected: weight.len(),
1294 actual: bias.len(),
1295 });
1296 }
1297 Ok(Self {
1298 groups,
1299 weight,
1300 bias,
1301 eps,
1302 })
1303 }
1304
1305 pub fn forward(&self, map: &mut FeatureMap) -> Result<(), NetworkError> {
1307 if map.channels != self.weight.len() {
1308 return Err(NetworkError::InvalidTarget {
1309 expected: self.weight.len(),
1310 actual: map.channels,
1311 });
1312 }
1313 let per_group = map.channels / self.groups;
1314 let pixels = map.pixels();
1315 let span = per_group * pixels;
1316
1317 map.data
1318 .par_chunks_mut(span)
1319 .enumerate()
1320 .for_each(|(group, values)| {
1321 let mean = values.iter().sum::<f32>() / span as f32;
1322 let variance = values
1323 .iter()
1324 .map(|value| (value - mean).powi(2))
1325 .sum::<f32>()
1326 / span as f32;
1327 let scale = (variance + self.eps).sqrt().recip();
1328 for (index, value) in values.iter_mut().enumerate() {
1329 let channel = group * per_group + index / pixels;
1330 *value = (*value - mean) * scale * self.weight[channel] + self.bias[channel];
1331 }
1332 });
1333 Ok(())
1334 }
1335}
1336
1337#[derive(Clone, Debug)]
1342pub struct TrainableGroupNorm {
1343 pub groups: usize,
1344 pub eps: f32,
1345 pub gamma: Param,
1346 pub beta: Param,
1347}
1348
1349#[derive(Clone, Debug)]
1351pub struct GroupNormCache {
1352 normalized: Matrix,
1353 inv_std: Vec<f32>,
1355}
1356
1357impl TrainableGroupNorm {
1358 pub fn new(groups: usize, channels: usize, eps: f32) -> Result<Self, NetworkError> {
1360 if groups == 0 || channels == 0 || channels % groups != 0 {
1361 return Err(NetworkError::InvalidConfig(format!(
1362 "{channels} channels do not divide into {groups} groups"
1363 )));
1364 }
1365 Ok(Self {
1366 groups,
1367 eps,
1368 gamma: Param::filled(1, channels, 1.0),
1369 beta: Param::zeros(1, channels),
1370 })
1371 }
1372
1373 pub fn forward(
1374 &self,
1375 input: &ImageBatch,
1376 ) -> Result<(ImageBatch, GroupNormCache), NetworkError> {
1377 let channels = self.gamma.value.cols;
1378 if input.channels() != channels {
1379 return Err(NetworkError::InvalidTarget {
1380 expected: channels,
1381 actual: input.channels(),
1382 });
1383 }
1384 let (per_group, pixels) = (channels / self.groups, input.pixels());
1385 let span = (per_group * pixels) as f32;
1386 let mut normalized = Matrix::new(input.tokens.rows, channels);
1387 let mut inv_std = vec![0f32; input.batch * self.groups];
1388 normalized
1389 .data
1390 .par_chunks_mut(pixels * channels)
1391 .zip(input.tokens.data.par_chunks(pixels * channels))
1392 .zip(inv_std.par_chunks_mut(self.groups))
1393 .for_each(|((out, sample), inv_std)| {
1394 for (group, inv_std) in inv_std.iter_mut().enumerate() {
1395 let range = group * per_group..(group + 1) * per_group;
1396 let values = || {
1397 range.clone().flat_map(|channel| {
1398 (0..pixels).map(move |p| sample[p * channels + channel])
1399 })
1400 };
1401 let mean = values().sum::<f32>() / span;
1402 let variance = values().map(|value| (value - mean).powi(2)).sum::<f32>() / span;
1403 *inv_std = (variance + self.eps).sqrt().recip();
1404 for p in 0..pixels {
1405 for channel in range.clone() {
1406 let at = p * channels + channel;
1407 out[at] = (sample[at] - mean) * *inv_std;
1408 }
1409 }
1410 }
1411 });
1412 let (gamma, beta) = (&self.gamma.value.data, &self.beta.value.data);
1413 let mut output = input.like(channels);
1414 for (out, row) in output
1415 .tokens
1416 .data
1417 .chunks_exact_mut(channels)
1418 .zip(normalized.data.chunks_exact(channels))
1419 {
1420 for channel in 0..channels {
1421 out[channel] = row[channel] * gamma[channel] + beta[channel];
1422 }
1423 }
1424 Ok((
1425 output,
1426 GroupNormCache {
1427 normalized,
1428 inv_std,
1429 },
1430 ))
1431 }
1432
1433 pub fn backward(&mut self, cache: &GroupNormCache, grad: &ImageBatch) -> ImageBatch {
1435 let channels = grad.channels();
1436 let (per_group, pixels) = (channels / self.groups, grad.pixels());
1437 let span = (per_group * pixels) as f32;
1438 for (g, row) in grad
1439 .tokens
1440 .data
1441 .chunks_exact(channels)
1442 .zip(cache.normalized.data.chunks_exact(channels))
1443 {
1444 for channel in 0..channels {
1445 self.gamma.grad.data[channel] += g[channel] * row[channel];
1446 self.beta.grad.data[channel] += g[channel];
1447 }
1448 }
1449 let gamma = &self.gamma.value.data;
1450 let mut input_grad = grad.like(channels);
1451 input_grad
1452 .tokens
1453 .data
1454 .par_chunks_mut(pixels * channels)
1455 .zip(grad.tokens.data.par_chunks(pixels * channels))
1456 .zip(cache.normalized.data.par_chunks(pixels * channels))
1457 .zip(cache.inv_std.par_chunks(self.groups))
1458 .for_each(|(((out, g), normalized), inv_std)| {
1459 for (group, &inv_std) in inv_std.iter().enumerate() {
1460 let range = group * per_group..(group + 1) * per_group;
1461 let (mut mean, mut along) = (0f32, 0f32);
1462 for p in 0..pixels {
1463 for channel in range.clone() {
1464 let at = p * channels + channel;
1465 let d = g[at] * gamma[channel];
1466 mean += d;
1467 along += d * normalized[at];
1468 }
1469 }
1470 let (mean, along) = (mean / span, along / span);
1471 for p in 0..pixels {
1472 for channel in range.clone() {
1473 let at = p * channels + channel;
1474 out[at] =
1475 inv_std * (g[at] * gamma[channel] - mean - normalized[at] * along);
1476 }
1477 }
1478 }
1479 });
1480 input_grad
1481 }
1482
1483 pub fn params_mut(&mut self) -> Vec<&mut Param> {
1484 vec![&mut self.gamma, &mut self.beta]
1485 }
1486
1487 pub fn to_cuda_on(&mut self, device: &crate::param::CudaDevice) -> Result<(), NetworkError> {
1490 self.gamma.to_cuda_on(device)?;
1491 self.beta.to_cuda_on(device)
1492 }
1493
1494 pub fn to_cpu(&mut self) -> Result<(), NetworkError> {
1495 self.gamma.to_cpu()?;
1496 self.beta.to_cpu()
1497 }
1498
1499 #[cfg(feature = "cuda")]
1501 pub fn forward_device(
1502 &self,
1503 input: &DeviceImageBatch,
1504 ) -> Result<(DeviceImageBatch, DeviceGroupNormCache), NetworkError> {
1505 let channels = self.gamma.value.cols;
1506 if input.channels != channels {
1507 return Err(NetworkError::InvalidTarget {
1508 expected: channels,
1509 actual: input.channels,
1510 });
1511 }
1512 let (gamma, beta) = match (&self.gamma.device, &self.beta.device) {
1513 (Some(gamma), Some(beta)) => (gamma.value(), beta.value()),
1514 _ => {
1515 return Err(NetworkError::InvalidConfig(
1516 "forward_device needs gamma and beta on a device".into(),
1517 ));
1518 }
1519 };
1520 let context = &input.context;
1521 let err = crate::cuda_training::cuda_err;
1522 let total = input.data.len();
1523 let blocks = input.batch * self.groups;
1524 let (mut mean, mut inv_std) = (context.zeros(blocks)?, context.zeros(blocks)?);
1525 let (mut output, mut normalized) = (context.zeros(total)?, context.zeros(total)?);
1526 let [n, pixels, channels, per_group] =
1527 [total, input.pixels(), channels, channels / self.groups].map(|v| v as i32);
1528 unsafe {
1529 context
1530 .stream
1531 .launch_builder(&context.image.gn_stats)
1532 .arg(&mut mean)
1533 .arg(&mut inv_std)
1534 .arg(&input.data)
1535 .arg(&pixels)
1536 .arg(&channels)
1537 .arg(&per_group)
1538 .arg(&self.eps)
1539 .launch(group_blocks(blocks))
1540 .map_err(err("group norm statistics kernel"))?;
1541 context
1542 .stream
1543 .launch_builder(&context.image.gn_apply)
1544 .arg(&mut output)
1545 .arg(&mut normalized)
1546 .arg(&input.data)
1547 .arg(&mean)
1548 .arg(&inv_std)
1549 .arg(gamma)
1550 .arg(beta)
1551 .arg(&n)
1552 .arg(&pixels)
1553 .arg(&channels)
1554 .arg(&per_group)
1555 .launch(crate::cuda_training::cfg(total))
1556 .map_err(err("group norm kernel"))?;
1557 }
1558 Ok((
1559 input.with_data(output),
1560 DeviceGroupNormCache {
1561 normalized,
1562 inv_std,
1563 },
1564 ))
1565 }
1566
1567 #[cfg(feature = "cuda")]
1570 pub fn backward_device(
1571 &mut self,
1572 cache: &DeviceGroupNormCache,
1573 grad: &DeviceImageBatch,
1574 ) -> Result<DeviceImageBatch, NetworkError> {
1575 let context = grad.context.clone();
1576 let err = crate::cuda_training::cuda_err;
1577 let (total, rows) = (grad.data.len(), grad.batch * grad.pixels());
1578 let blocks = grad.batch * self.groups;
1579 let channels = self.gamma.value.cols;
1580 let (Some(gamma), Some(beta)) = (self.gamma.device.as_mut(), self.beta.device.as_mut())
1581 else {
1582 return Err(NetworkError::InvalidConfig(
1583 "backward_device needs gamma and beta on a device".into(),
1584 ));
1585 };
1586 let ones = context.filled(rows, 1.0)?;
1587 let mut product = context.zeros(total)?;
1588 let (mut first, mut second) = (context.zeros(blocks)?, context.zeros(blocks)?);
1589 let mut input_grad = context.zeros(total)?;
1590 let [n, pixels, channels, per_group] =
1591 [total, grad.pixels(), channels, channels / self.groups].map(|v| v as i32);
1592 unsafe {
1593 context
1594 .stream
1595 .launch_builder(&context.image.mul)
1596 .arg(&mut product)
1597 .arg(&grad.data)
1598 .arg(&cache.normalized)
1599 .arg(&n)
1600 .launch(crate::cuda_training::cfg(total))
1601 .map_err(err("multiply kernel"))?;
1602 context
1603 .stream
1604 .launch_builder(&context.image.gn_bwd_stats)
1605 .arg(&mut first)
1606 .arg(&mut second)
1607 .arg(&grad.data)
1608 .arg(&cache.normalized)
1609 .arg(gamma.value())
1610 .arg(&pixels)
1611 .arg(&channels)
1612 .arg(&per_group)
1613 .launch(group_blocks(blocks))
1614 .map_err(err("group norm backward statistics kernel"))?;
1615 context
1616 .stream
1617 .launch_builder(&context.image.gn_bwd_apply)
1618 .arg(&mut input_grad)
1619 .arg(&grad.data)
1620 .arg(&cache.normalized)
1621 .arg(&cache.inv_std)
1622 .arg(&first)
1623 .arg(&second)
1624 .arg(gamma.value())
1625 .arg(&n)
1626 .arg(&pixels)
1627 .arg(&channels)
1628 .arg(&per_group)
1629 .launch(crate::cuda_training::cfg(total))
1630 .map_err(err("group norm backward kernel"))?;
1631 }
1632 gamma.accumulate_grad_device(&ones, &product, rows)?;
1633 beta.accumulate_grad_device(&ones, &grad.data, rows)?;
1634 Ok(grad.with_data(input_grad))
1635 }
1636}
1637
1638#[cfg(feature = "cuda")]
1640pub struct DeviceGroupNormCache {
1641 normalized: cudarc::driver::CudaSlice<f32>,
1642 inv_std: cudarc::driver::CudaSlice<f32>,
1643}
1644
1645#[cfg(feature = "cuda")]
1647fn group_blocks(blocks: usize) -> cudarc::driver::LaunchConfig {
1648 cudarc::driver::LaunchConfig {
1649 grid_dim: (blocks as u32, 1, 1),
1650 block_dim: (256, 1, 1),
1651 shared_mem_bytes: 0,
1652 }
1653}
1654
1655#[cfg(feature = "cuda")]
1658pub fn silu_device(input: &DeviceImageBatch) -> Result<DeviceImageBatch, NetworkError> {
1659 silu_launch(input, None)
1660}
1661
1662#[cfg(feature = "cuda")]
1664pub fn silu_backward_device(
1665 input: &DeviceImageBatch,
1666 grad: &DeviceImageBatch,
1667) -> Result<DeviceImageBatch, NetworkError> {
1668 silu_launch(input, Some(grad))
1669}
1670
1671#[cfg(feature = "cuda")]
1673pub struct DeviceSpatialSoftmaxCache {
1674 probabilities: DeviceImageBatch,
1675 expected: cudarc::driver::CudaSlice<f32>,
1676}
1677
1678#[cfg(feature = "cuda")]
1681pub fn spatial_softmax_device(
1682 input: &DeviceImageBatch,
1683) -> Result<(DeviceImageBatch, DeviceSpatialSoftmaxCache), NetworkError> {
1684 let context = &input.context;
1685 let channels = input.channels;
1686 let mut probabilities = context.zeros(input.data.len())?;
1687 let mut expected = context.zeros(input.batch * 2 * channels)?;
1688 let [height, width, channels_arg] = [input.height, input.width, channels].map(|v| v as i32);
1689 unsafe {
1690 context
1691 .stream
1692 .launch_builder(&context.image.ssm_fwd)
1693 .arg(&mut probabilities)
1694 .arg(&mut expected)
1695 .arg(&input.data)
1696 .arg(&height)
1697 .arg(&width)
1698 .arg(&channels_arg)
1699 .launch(group_blocks(input.batch * channels))
1700 .map_err(crate::cuda_training::cuda_err("spatial softmax kernel"))?;
1701 }
1702 let output = DeviceImageBatch {
1703 batch: input.batch,
1704 height: 1,
1705 width: 1,
1706 channels: 2 * channels,
1707 data: expected.clone(),
1708 context: context.clone(),
1709 };
1710 let probabilities = input.with_data(probabilities);
1711 Ok((
1712 output,
1713 DeviceSpatialSoftmaxCache {
1714 probabilities,
1715 expected,
1716 },
1717 ))
1718}
1719
1720#[cfg(feature = "cuda")]
1722pub fn spatial_softmax_backward_device(
1723 cache: &DeviceSpatialSoftmaxCache,
1724 grad: &DeviceImageBatch,
1725) -> Result<DeviceImageBatch, NetworkError> {
1726 let probabilities = &cache.probabilities;
1727 let context = &probabilities.context;
1728 let total = probabilities.data.len();
1729 let mut out = context.zeros(total)?;
1730 let [n, height, width, channels] = [
1731 total,
1732 probabilities.height,
1733 probabilities.width,
1734 probabilities.channels,
1735 ]
1736 .map(|v| v as i32);
1737 unsafe {
1738 context
1739 .stream
1740 .launch_builder(&context.image.ssm_bwd)
1741 .arg(&mut out)
1742 .arg(&probabilities.data)
1743 .arg(&grad.data)
1744 .arg(&cache.expected)
1745 .arg(&n)
1746 .arg(&height)
1747 .arg(&width)
1748 .arg(&channels)
1749 .launch(crate::cuda_training::cfg(total))
1750 .map_err(crate::cuda_training::cuda_err(
1751 "spatial softmax backward kernel",
1752 ))?;
1753 }
1754 Ok(probabilities.with_data(out))
1755}
1756
1757#[cfg(feature = "cuda")]
1759pub fn global_avg_pool_device(input: &DeviceImageBatch) -> Result<DeviceImageBatch, NetworkError> {
1760 let context = &input.context;
1761 let mut out = context.zeros(input.batch * input.channels)?;
1762 let [pixels, channels] = [input.pixels(), input.channels].map(|v| v as i32);
1763 unsafe {
1764 context
1765 .stream
1766 .launch_builder(&context.image.gap_fwd)
1767 .arg(&mut out)
1768 .arg(&input.data)
1769 .arg(&pixels)
1770 .arg(&channels)
1771 .launch(group_blocks(input.batch * input.channels))
1772 .map_err(crate::cuda_training::cuda_err("average pool kernel"))?;
1773 }
1774 Ok(DeviceImageBatch {
1775 height: 1,
1776 width: 1,
1777 ..input.with_data(out)
1778 })
1779}
1780
1781#[cfg(feature = "cuda")]
1784pub fn global_avg_pool_backward_device(
1785 grad: &DeviceImageBatch,
1786 height: usize,
1787 width: usize,
1788) -> Result<DeviceImageBatch, NetworkError> {
1789 let context = &grad.context;
1790 let total = grad.batch * height * width * grad.channels;
1791 let mut out = context.zeros(total)?;
1792 let [n, pixels, channels] = [total, height * width, grad.channels].map(|v| v as i32);
1793 unsafe {
1794 context
1795 .stream
1796 .launch_builder(&context.image.gap_bwd)
1797 .arg(&mut out)
1798 .arg(&grad.data)
1799 .arg(&n)
1800 .arg(&pixels)
1801 .arg(&channels)
1802 .launch(crate::cuda_training::cfg(total))
1803 .map_err(crate::cuda_training::cuda_err(
1804 "average pool backward kernel",
1805 ))?;
1806 }
1807 Ok(DeviceImageBatch {
1808 height,
1809 width,
1810 ..grad.with_data(out)
1811 })
1812}
1813
1814#[cfg(feature = "cuda")]
1815fn silu_launch(
1816 input: &DeviceImageBatch,
1817 grad: Option<&DeviceImageBatch>,
1818) -> Result<DeviceImageBatch, NetworkError> {
1819 let context = &input.context;
1820 let total = input.data.len();
1821 let mut output = context.uninit(total)?;
1822 let n = total as i32;
1823 let mut launch = match grad {
1824 None => context.stream.launch_builder(&context.image.silu),
1825 Some(_) => context.stream.launch_builder(&context.image.silu_bwd),
1826 };
1827 launch.arg(&mut output);
1828 if let Some(grad) = grad {
1829 launch.arg(&grad.data);
1830 }
1831 launch.arg(&input.data).arg(&n);
1832 unsafe { launch.launch(crate::cuda_training::cfg(total)) }
1833 .map_err(crate::cuda_training::cuda_err("SiLU kernel"))?;
1834 Ok(input.with_data(output))
1835}
1836
1837pub fn silu(map: &mut FeatureMap) {
1839 map.data
1840 .par_iter_mut()
1841 .for_each(|value| *value = crate::ffn::silu(*value));
1842}
1843
1844pub fn upsample_nearest(map: &FeatureMap, factor: usize) -> FeatureMap {
1847 let (height, width) = (map.height * factor, map.width * factor);
1848 let mut output = FeatureMap::new(map.channels, height, width);
1849 let pixels = height * width;
1850 output
1851 .data
1852 .par_chunks_mut(pixels)
1853 .enumerate()
1854 .for_each(|(channel, plane)| {
1855 let source = map.plane(channel);
1856 for (pixel, value) in plane.iter_mut().enumerate() {
1857 let (y, x) = (pixel / width / factor, pixel % width / factor);
1858 *value = source[y * map.width + x];
1859 }
1860 });
1861 output
1862}
1863
1864pub fn pixel_shuffle(map: &FeatureMap, factor: usize) -> Result<FeatureMap, NetworkError> {
1870 let square = factor * factor;
1871 if factor == 0 || map.channels % square != 0 {
1872 return Err(NetworkError::InvalidConfig(format!(
1873 "{} channels do not shuffle by {factor}",
1874 map.channels
1875 )));
1876 }
1877 let (channels, height, width) = (
1878 map.channels / square,
1879 map.height * factor,
1880 map.width * factor,
1881 );
1882 let mut output = FeatureMap::new(channels, height, width);
1883 let pixels = height * width;
1884 output
1885 .data
1886 .par_chunks_mut(pixels)
1887 .enumerate()
1888 .for_each(|(channel, plane)| {
1889 for (pixel, value) in plane.iter_mut().enumerate() {
1890 let (y, x) = (pixel / width, pixel % width);
1891 let source = (channel * square + (y % factor) * factor + x % factor)
1892 * map.height
1893 * map.width
1894 + (y / factor) * map.width
1895 + x / factor;
1896 *value = map.data[source];
1897 }
1898 });
1899 Ok(output)
1900}
1901
1902pub fn pixel_unshuffle(map: &FeatureMap, factor: usize) -> Result<FeatureMap, NetworkError> {
1905 if factor == 0 || map.height % factor != 0 || map.width % factor != 0 {
1906 return Err(NetworkError::InvalidConfig(format!(
1907 "a {}x{} map does not unshuffle by {factor}",
1908 map.height, map.width
1909 )));
1910 }
1911 let square = factor * factor;
1912 let (channels, height, width) = (
1913 map.channels * square,
1914 map.height / factor,
1915 map.width / factor,
1916 );
1917 let mut output = FeatureMap::new(channels, height, width);
1918 let pixels = height * width;
1919 output
1920 .data
1921 .par_chunks_mut(pixels)
1922 .enumerate()
1923 .for_each(|(channel, plane)| {
1924 let (source_channel, offset) = (channel / square, channel % square);
1925 let (row_offset, column_offset) = (offset / factor, offset % factor);
1926 let source = map.plane(source_channel);
1927 for (pixel, value) in plane.iter_mut().enumerate() {
1928 let (y, x) = (pixel / width, pixel % width);
1929 *value = source[(y * factor + row_offset) * map.width + x * factor + column_offset];
1930 }
1931 });
1932 Ok(output)
1933}
1934
1935fn unit_coordinates(n: usize) -> Vec<f32> {
1937 match n {
1938 1 => vec![0.0],
1939 _ => (0..n)
1940 .map(|i| -1.0 + 2.0 * i as f32 / (n - 1) as f32)
1941 .collect(),
1942 }
1943}
1944
1945pub struct SpatialSoftmaxCache {
1947 probabilities: ImageBatch,
1950 expected: Matrix,
1952}
1953
1954pub fn spatial_softmax(input: &ImageBatch) -> (Matrix, SpatialSoftmaxCache) {
1959 let (channels, pixels, width) = (input.channels(), input.pixels(), input.width);
1960 let (xs, ys) = (
1961 unit_coordinates(input.width),
1962 unit_coordinates(input.height),
1963 );
1964 let mut probabilities = input.like(channels);
1965 let mut expected = Matrix::new(input.batch, 2 * channels);
1966 probabilities
1967 .tokens
1968 .data
1969 .par_chunks_mut(pixels * channels)
1970 .zip(input.tokens.data.par_chunks(pixels * channels))
1971 .zip(expected.data.par_chunks_mut(2 * channels))
1972 .for_each(|((probability, logits), expected)| {
1973 for channel in 0..channels {
1974 let logit = |pixel: usize| logits[pixel * channels + channel];
1975 let max = (0..pixels).map(logit).fold(f32::NEG_INFINITY, f32::max);
1976 let mut sum = 0.0;
1977 for pixel in 0..pixels {
1978 let value = (logit(pixel) - max).exp();
1979 probability[pixel * channels + channel] = value;
1980 sum += value;
1981 }
1982 let (mut x, mut y) = (0.0, 0.0);
1983 for pixel in 0..pixels {
1984 let slot = &mut probability[pixel * channels + channel];
1985 *slot /= sum;
1986 x += *slot * xs[pixel % width];
1987 y += *slot * ys[pixel / width];
1988 }
1989 (expected[channel], expected[channels + channel]) = (x, y);
1990 }
1991 });
1992 let cache = SpatialSoftmaxCache {
1993 probabilities,
1994 expected: expected.clone(),
1995 };
1996 (expected, cache)
1997}
1998
1999pub fn spatial_softmax_backward(cache: &SpatialSoftmaxCache, grad: &Matrix) -> ImageBatch {
2001 let probabilities = &cache.probabilities;
2002 let (channels, pixels, width) = (
2003 probabilities.channels(),
2004 probabilities.pixels(),
2005 probabilities.width,
2006 );
2007 let (xs, ys) = (
2008 unit_coordinates(probabilities.width),
2009 unit_coordinates(probabilities.height),
2010 );
2011 let mut out = probabilities.like(channels);
2012 out.tokens
2013 .data
2014 .par_chunks_mut(pixels * channels)
2015 .zip(probabilities.tokens.data.par_chunks(pixels * channels))
2016 .zip(
2017 grad.data
2018 .par_chunks(2 * channels)
2019 .zip(cache.expected.data.par_chunks(2 * channels)),
2020 )
2021 .for_each(|((out, probability), (grad, expected))| {
2022 for pixel in 0..pixels {
2023 for channel in 0..channels {
2024 let (gx, gy) = (grad[channel], grad[channels + channel]);
2027 let along = gx * (xs[pixel % width] - expected[channel])
2028 + gy * (ys[pixel / width] - expected[channels + channel]);
2029 let slot = pixel * channels + channel;
2030 out[slot] = probability[slot] * along;
2031 }
2032 }
2033 });
2034 out
2035}
2036
2037pub fn global_avg_pool(input: &ImageBatch) -> Matrix {
2039 let (channels, pixels) = (input.channels(), input.pixels());
2040 let mut out = Matrix::new(input.batch, channels);
2041 out.data
2042 .par_chunks_mut(channels)
2043 .zip(input.tokens.data.par_chunks(pixels * channels))
2044 .for_each(|(out, sample)| {
2045 for pixel in sample.chunks_exact(channels) {
2046 out.iter_mut()
2047 .zip(pixel)
2048 .for_each(|(sum, value)| *sum += value);
2049 }
2050 out.iter_mut().for_each(|sum| *sum /= pixels as f32);
2051 });
2052 out
2053}
2054
2055pub fn global_avg_pool_backward(grad: &Matrix, height: usize, width: usize) -> ImageBatch {
2057 let (channels, pixels) = (grad.cols, height * width);
2058 let mut out = ImageBatch::new(grad.rows, height, width, channels);
2059 out.tokens
2060 .data
2061 .par_chunks_mut(pixels * channels)
2062 .zip(grad.data.par_chunks(channels))
2063 .for_each(|(out, grad)| {
2064 for pixel in out.chunks_exact_mut(channels) {
2065 pixel
2066 .iter_mut()
2067 .zip(grad)
2068 .for_each(|(slot, value)| *slot = value / pixels as f32);
2069 }
2070 });
2071 out
2072}
2073
2074#[derive(Clone, Debug)]
2078pub struct TrainableBatchNorm2d {
2079 pub eps: f32,
2080 pub momentum: f32,
2082 pub gamma: Param,
2083 pub beta: Param,
2084 pub running: Param,
2088}
2089
2090#[derive(Clone, Debug)]
2092pub struct BatchNormCache {
2093 normalized: Matrix,
2094 inv_std: Vec<f32>,
2095 training: bool,
2096}
2097
2098impl TrainableBatchNorm2d {
2099 pub fn new(channels: usize, eps: f32, momentum: f32) -> Self {
2101 let mut running = Matrix::new(2, channels);
2102 running.data[channels..].fill(1.0);
2103 Self {
2104 eps,
2105 momentum,
2106 gamma: Param::filled(1, channels, 1.0),
2107 beta: Param::zeros(1, channels),
2108 running: Param::new(running),
2109 }
2110 }
2111
2112 pub fn yolo(channels: usize) -> Self {
2114 Self::new(channels, 1e-3, 0.03)
2115 }
2116
2117 pub fn channels(&self) -> usize {
2118 self.gamma.value.cols
2119 }
2120
2121 pub fn forward(
2124 &mut self,
2125 input: &ImageBatch,
2126 training: bool,
2127 ) -> Result<(ImageBatch, BatchNormCache), NetworkError> {
2128 let channels = self.channels();
2129 if input.channels() != channels {
2130 return Err(NetworkError::InvalidTarget {
2131 expected: channels,
2132 actual: input.channels(),
2133 });
2134 }
2135 let rows = input.tokens.rows;
2136 let (mut mean, mut variance) = (vec![0f32; channels], vec![0f32; channels]);
2137 if training {
2138 for row in input.tokens.data.chunks_exact(channels) {
2139 mean.iter_mut().zip(row).for_each(|(m, v)| *m += v);
2140 }
2141 mean.iter_mut().for_each(|m| *m /= rows as f32);
2142 for row in input.tokens.data.chunks_exact(channels) {
2143 for c in 0..channels {
2144 variance[c] += (row[c] - mean[c]).powi(2);
2145 }
2146 }
2147 variance.iter_mut().for_each(|v| *v /= rows as f32);
2148 let unbias = rows as f32 / (rows as f32 - 1.0).max(1.0);
2149 let m = self.momentum;
2150 let running = &mut self.running.value.data;
2151 for c in 0..channels {
2152 running[c] = (1.0 - m) * running[c] + m * mean[c];
2153 running[channels + c] =
2154 (1.0 - m) * running[channels + c] + m * variance[c] * unbias;
2155 }
2156 } else {
2157 mean.copy_from_slice(&self.running.value.data[..channels]);
2158 variance.copy_from_slice(&self.running.value.data[channels..]);
2159 }
2160 let inv_std: Vec<f32> = variance
2161 .iter()
2162 .map(|v| 1.0 / (v + self.eps).sqrt())
2163 .collect();
2164 let mut normalized = Matrix::new(rows, channels);
2165 let mut output = input.like(channels);
2166 let (gamma, beta) = (&self.gamma.value.data, &self.beta.value.data);
2167 for ((out, h), row) in output
2168 .tokens
2169 .data
2170 .chunks_exact_mut(channels)
2171 .zip(normalized.data.chunks_exact_mut(channels))
2172 .zip(input.tokens.data.chunks_exact(channels))
2173 {
2174 for c in 0..channels {
2175 h[c] = (row[c] - mean[c]) * inv_std[c];
2176 out[c] = h[c] * gamma[c] + beta[c];
2177 }
2178 }
2179 Ok((
2180 output,
2181 BatchNormCache {
2182 normalized,
2183 inv_std,
2184 training,
2185 },
2186 ))
2187 }
2188
2189 pub fn backward(&mut self, cache: &BatchNormCache, grad: &ImageBatch) -> ImageBatch {
2191 let channels = self.channels();
2192 let rows = grad.tokens.rows as f32;
2193 let (mut mean, mut along) = (vec![0f32; channels], vec![0f32; channels]);
2194 for (g, h) in grad
2195 .tokens
2196 .data
2197 .chunks_exact(channels)
2198 .zip(cache.normalized.data.chunks_exact(channels))
2199 {
2200 for c in 0..channels {
2201 mean[c] += g[c];
2202 along[c] += g[c] * h[c];
2203 }
2204 }
2205 for c in 0..channels {
2206 self.gamma.grad.data[c] += along[c];
2207 self.beta.grad.data[c] += mean[c];
2208 }
2209 let gamma = &self.gamma.value.data;
2210 let mut input_grad = grad.like(channels);
2211 for ((out, g), h) in input_grad
2212 .tokens
2213 .data
2214 .chunks_exact_mut(channels)
2215 .zip(grad.tokens.data.chunks_exact(channels))
2216 .zip(cache.normalized.data.chunks_exact(channels))
2217 {
2218 for c in 0..channels {
2219 let centred = match cache.training {
2220 true => g[c] - (mean[c] + h[c] * along[c]) / rows,
2221 false => g[c],
2222 };
2223 out[c] = cache.inv_std[c] * gamma[c] * centred;
2224 }
2225 }
2226 input_grad
2227 }
2228
2229 pub fn params_mut(&mut self) -> Vec<&mut Param> {
2230 vec![&mut self.gamma, &mut self.beta]
2231 }
2232
2233 pub fn to_cuda_on(&mut self, device: &crate::param::CudaDevice) -> Result<(), NetworkError> {
2235 self.gamma.to_cuda_on(device)?;
2236 self.beta.to_cuda_on(device)?;
2237 self.running.to_cuda_on(device)
2238 }
2239
2240 pub fn to_cpu(&mut self) -> Result<(), NetworkError> {
2241 self.gamma.to_cpu()?;
2242 self.beta.to_cpu()?;
2243 self.running.to_cpu()
2244 }
2245
2246 pub fn fuse_into(&self, conv: &TrainableConv2d) -> TrainableConv2d {
2250 let channels = self.channels();
2251 let mut fused = conv.clone();
2252 let patch = fused.weight.weight.value.cols;
2253 let running = &self.running.value.data;
2254 for c in 0..channels {
2255 let scale = self.gamma.value.data[c] / (running[channels + c] + self.eps).sqrt();
2256 fused.weight.weight.value.data[c * patch..(c + 1) * patch]
2257 .iter_mut()
2258 .for_each(|w| *w *= scale);
2259 let bias = if conv.has_bias {
2260 conv.bias.value.data[c]
2261 } else {
2262 0.0
2263 };
2264 fused.bias.value.data[c] = (bias - running[c]) * scale + self.beta.value.data[c];
2265 }
2266 fused.has_bias = true;
2267 fused
2268 }
2269
2270 #[cfg(feature = "cuda")]
2272 pub fn forward_device(
2273 &mut self,
2274 input: &DeviceImageBatch,
2275 training: bool,
2276 ) -> Result<(DeviceImageBatch, DeviceBatchNormCache), NetworkError> {
2277 self.forward_device_with(input.clone(), training, false)
2278 }
2279
2280 #[cfg(feature = "cuda")]
2284 pub(crate) fn forward_device_with(
2285 &mut self,
2286 input: DeviceImageBatch,
2287 training: bool,
2288 silu: bool,
2289 ) -> Result<(DeviceImageBatch, DeviceBatchNormCache), NetworkError> {
2290 let channels = self.channels();
2291 if input.channels != channels {
2292 return Err(NetworkError::InvalidTarget {
2293 expected: channels,
2294 actual: input.channels,
2295 });
2296 }
2297 let (Some(gamma), Some(beta), Some(running)) = (
2298 self.gamma.device.as_ref(),
2299 self.beta.device.as_ref(),
2300 self.running.device.as_mut(),
2301 ) else {
2302 return Err(NetworkError::InvalidConfig(
2303 "forward_device needs the batch norm on a device".into(),
2304 ));
2305 };
2306 let context = &input.context;
2307 let err = crate::cuda_training::cuda_err;
2308 let (total, rows) = (input.data.len(), input.batch * input.pixels());
2309 let (mut mean, mut variance) = (context.zeros(channels)?, context.zeros(channels)?);
2310 let mut inv_std = context.zeros(channels)?;
2311 if training {
2312 let scale = 1.0 / rows as f32;
2313 let shift = ColumnSum::Shifted(&input.data, &mut variance);
2316 column_sums(
2317 context,
2318 &mut mean,
2319 &input.data,
2320 Some(shift),
2321 rows,
2322 channels,
2323 scale,
2324 )?;
2325 }
2326 let unbias = rows as f32 / (rows as f32 - 1.0).max(1.0);
2327 let [c, train] = [channels as i32, training as i32];
2328 unsafe {
2329 context
2330 .stream
2331 .launch_builder(&context.image.bn_finish)
2332 .arg(&mut mean)
2333 .arg(&mut variance)
2334 .arg(&mut inv_std)
2335 .arg(running.value_mut())
2336 .arg(&input.data)
2337 .arg(&c)
2338 .arg(&self.eps)
2339 .arg(&self.momentum)
2340 .arg(&unbias)
2341 .arg(&train)
2342 .launch(crate::cuda_training::cfg(channels))
2343 .map_err(err("batch norm statistics kernel"))?;
2344 }
2345 let mut output = context.uninit(total)?;
2346 let n = total as i32;
2347 unsafe {
2348 context
2349 .stream
2350 .launch_builder(&context.image.bn_apply)
2351 .arg(&mut output)
2352 .arg(&input.data)
2353 .arg(&mean)
2354 .arg(&inv_std)
2355 .arg(gamma.value())
2356 .arg(beta.value())
2357 .arg(&n)
2358 .arg(&c)
2359 .arg(&(silu as i32))
2360 .launch(crate::cuda_training::cfg(total))
2361 .map_err(err("batch norm kernel"))?;
2362 }
2363 Ok((
2364 input.with_data(output),
2365 DeviceBatchNormCache {
2366 input: input.data,
2367 mean,
2368 inv_std,
2369 training,
2370 silu,
2371 },
2372 ))
2373 }
2374
2375 #[cfg(feature = "cuda")]
2377 pub fn backward_device(
2378 &mut self,
2379 cache: &DeviceBatchNormCache,
2380 grad: &DeviceImageBatch,
2381 ) -> Result<DeviceImageBatch, NetworkError> {
2382 let context = grad.context.clone();
2383 let err = crate::cuda_training::cuda_err;
2384 let (total, rows) = (grad.data.len(), grad.batch * grad.pixels());
2385 let channels = self.channels();
2386 let (Some(gamma), Some(beta)) = (self.gamma.device.as_mut(), self.beta.device.as_mut())
2387 else {
2388 return Err(NetworkError::InvalidConfig(
2389 "backward_device needs gamma and beta on a device".into(),
2390 ));
2391 };
2392 let (mut mean, mut along) = (context.zeros(channels)?, context.zeros(channels)?);
2395 let silu = cache.silu.then_some((gamma.value(), beta.value()));
2396 let times = ColumnSum::Times(
2397 &cache.input,
2398 &mut along,
2399 (&cache.mean, &cache.inv_std),
2400 silu,
2401 );
2402 let scale = 1.0 / rows as f32;
2403 column_sums(
2404 &context,
2405 &mut mean,
2406 &grad.data,
2407 Some(times),
2408 rows,
2409 channels,
2410 scale,
2411 )?;
2412 let unused = context.zeros(channels)?;
2413 let (centre, slope) = match cache.training {
2414 true => (&mean, &along),
2415 false => (&unused, &unused),
2416 };
2417 let mut input_grad = context.uninit(total)?;
2418 let [n, c] = [total, channels].map(|v| v as i32);
2419 unsafe {
2420 context
2421 .stream
2422 .launch_builder(&context.image.bn_bwd_apply)
2423 .arg(&mut input_grad)
2424 .arg(&grad.data)
2425 .arg(&cache.input)
2426 .arg(&cache.mean)
2427 .arg(&cache.inv_std)
2428 .arg(centre)
2429 .arg(slope)
2430 .arg(gamma.value())
2431 .arg(beta.value())
2432 .arg(&n)
2433 .arg(&c)
2434 .arg(&(cache.silu as i32))
2435 .launch(crate::cuda_training::cfg(total))
2436 .map_err(err("batch norm backward kernel"))?;
2437 }
2438 let count = context.filled(1, rows as f32)?;
2439 gamma.accumulate_grad_device(&count, &along, 1)?;
2440 beta.accumulate_grad_device(&count, &mean, 1)?;
2441 Ok(grad.with_data(input_grad))
2442 }
2443}
2444
2445#[cfg(feature = "cuda")]
2447pub struct DeviceBatchNormCache {
2448 input: cudarc::driver::CudaSlice<f32>,
2451 mean: cudarc::driver::CudaSlice<f32>,
2452 inv_std: cudarc::driver::CudaSlice<f32>,
2453 training: bool,
2454 silu: bool,
2455}
2456
2457#[cfg(feature = "cuda")]
2459enum ColumnSum<'a> {
2460 Shifted(
2463 &'a cudarc::driver::CudaSlice<f32>,
2464 &'a mut cudarc::driver::CudaSlice<f32>,
2465 ),
2466 Times(
2471 &'a cudarc::driver::CudaSlice<f32>,
2472 &'a mut cudarc::driver::CudaSlice<f32>,
2473 (
2474 &'a cudarc::driver::CudaSlice<f32>,
2475 &'a cudarc::driver::CudaSlice<f32>,
2476 ),
2477 Option<(
2478 &'a cudarc::driver::CudaSlice<f32>,
2479 &'a cudarc::driver::CudaSlice<f32>,
2480 )>,
2481 ),
2482}
2483
2484#[cfg(feature = "cuda")]
2487fn column_sums(
2488 context: &crate::gpu_transformer::GpuContext,
2489 out: &mut cudarc::driver::CudaSlice<f32>,
2490 x: &cudarc::driver::CudaSlice<f32>,
2491 with: Option<ColumnSum>,
2492 rows: usize,
2493 channels: usize,
2494 scale: f32,
2495) -> Result<(), NetworkError> {
2496 let [r, c] = [rows, channels].map(|v| v as i32);
2497 let mut launch = context.stream.launch_builder(&context.image.col_sums);
2498 launch.arg(out);
2499 let with_silu = match &with {
2501 Some(ColumnSum::Times(_, _, _, silu)) => *silu,
2502 _ => None,
2503 };
2504 let stats = match &with {
2505 Some(ColumnSum::Times(_, _, stats, _)) => *stats,
2506 _ => (x, x),
2507 };
2508 let mode = match with {
2509 None => {
2510 launch.arg(x).arg(x).arg(x);
2511 0
2512 }
2513 Some(ColumnSum::Shifted(centre, second)) => {
2514 launch.arg(second).arg(x).arg(centre);
2515 1
2516 }
2517 Some(ColumnSum::Times(other, second, _, _)) => {
2518 launch.arg(second).arg(x).arg(other);
2519 2
2520 }
2521 };
2522 launch.arg(&r).arg(&c).arg(&scale).arg(&mode);
2523 let (gamma, beta) = match with_silu {
2524 Some(pair) => pair,
2525 None => (x, x),
2526 };
2527 let act = with_silu.is_some() as i32;
2528 launch
2529 .arg(gamma)
2530 .arg(beta)
2531 .arg(&act)
2532 .arg(stats.0)
2533 .arg(stats.1);
2534 unsafe { launch.launch(crate::cuda_training::cfg(rows.div_ceil(64) * channels)) }
2536 .map_err(crate::cuda_training::cuda_err("column sum kernel"))?;
2537 Ok(())
2538}
2539
2540#[derive(Clone, Debug)]
2544pub struct DepthwiseConv2d {
2545 pub weight: Param,
2547 kernel: usize,
2548 stride: usize,
2549}
2550
2551#[derive(Clone, Debug)]
2553pub struct DepthwiseCache {
2554 input: ImageBatch,
2555}
2556
2557impl DepthwiseConv2d {
2558 pub fn new(channels: usize, kernel: usize, stride: usize, rng: &mut StdRng) -> Self {
2559 let fan_in = kernel * kernel;
2560 Self {
2561 weight: Param::he_uniform(1, channels * fan_in, fan_in, rng),
2562 kernel,
2563 stride,
2564 }
2565 }
2566
2567 pub fn channels(&self) -> usize {
2568 self.weight.value.cols / (self.kernel * self.kernel)
2569 }
2570
2571 pub fn output_size(&self, height: usize, width: usize) -> (usize, usize) {
2572 let size = |n: usize| (n + 2 * (self.kernel / 2) - self.kernel) / self.stride + 1;
2573 (size(height), size(width))
2574 }
2575
2576 fn taps(&self, input: &ImageBatch) -> impl Iterator<Item = (usize, usize, usize)> + use<> {
2579 let (h, w, c) = (input.height, input.width, input.channels());
2580 let (oh, ow) = self.output_size(h, w);
2581 let (k, s, pad) = (self.kernel, self.stride, self.kernel / 2);
2582 let batch = input.batch;
2583 (0..batch * oh * ow).flat_map(move |row| {
2584 let (ox, oy, b) = (row % ow, row / ow % oh, row / ow / oh);
2585 (0..k * k).flat_map(move |t| {
2586 let (sy, sx) = (
2587 (oy * s + t / k).wrapping_sub(pad),
2588 (ox * s + t % k).wrapping_sub(pad),
2589 );
2590 let inside = sy < h && sx < w;
2591 (0..c).filter(move |_| inside).map(move |ch| {
2592 (
2593 row * c + ch,
2594 ((b * h + sy) * w + sx) * c + ch,
2595 ch * k * k + t,
2596 )
2597 })
2598 })
2599 })
2600 }
2601
2602 pub fn forward(
2603 &self,
2604 input: &ImageBatch,
2605 ) -> Result<(ImageBatch, DepthwiseCache), NetworkError> {
2606 if input.channels() != self.channels() {
2607 return Err(NetworkError::InvalidTarget {
2608 expected: self.channels(),
2609 actual: input.channels(),
2610 });
2611 }
2612 let (oh, ow) = self.output_size(input.height, input.width);
2613 let mut output = ImageBatch::new(input.batch, oh, ow, input.channels());
2614 let weight = &self.weight.value.data;
2615 for (o, i, t) in self.taps(input) {
2616 output.tokens.data[o] += input.tokens.data[i] * weight[t];
2617 }
2618 Ok((
2619 output,
2620 DepthwiseCache {
2621 input: input.clone(),
2622 },
2623 ))
2624 }
2625
2626 pub fn backward(&mut self, cache: &DepthwiseCache, grad: &ImageBatch) -> ImageBatch {
2628 let mut input_grad = cache.input.like(cache.input.channels());
2629 let weight = self.weight.value.data.clone();
2630 for (o, i, t) in self.taps(&cache.input) {
2631 input_grad.tokens.data[i] += grad.tokens.data[o] * weight[t];
2632 self.weight.grad.data[t] += grad.tokens.data[o] * cache.input.tokens.data[i];
2633 }
2634 input_grad
2635 }
2636
2637 pub fn params_mut(&mut self) -> Vec<&mut Param> {
2638 vec![&mut self.weight]
2639 }
2640
2641 pub fn to_cuda_on(&mut self, device: &crate::param::CudaDevice) -> Result<(), NetworkError> {
2642 self.weight.to_cuda_on(device)
2643 }
2644
2645 pub fn to_cpu(&mut self) -> Result<(), NetworkError> {
2646 self.weight.to_cpu()
2647 }
2648
2649 #[cfg(feature = "cuda")]
2652 pub fn forward_device(
2653 &self,
2654 input: &DeviceImageBatch,
2655 ) -> Result<DeviceImageBatch, NetworkError> {
2656 let Some(weight) = self.weight.device.as_ref() else {
2657 return Err(NetworkError::InvalidConfig(
2658 "forward_device needs the weight on a device".into(),
2659 ));
2660 };
2661 if input.channels != self.channels() {
2662 return Err(NetworkError::InvalidTarget {
2663 expected: self.channels(),
2664 actual: input.channels,
2665 });
2666 }
2667 let (oh, ow) = self.output_size(input.height, input.width);
2668 let context = &input.context;
2669 let total = input.batch * oh * ow * input.channels;
2670 let mut out = context.uninit(total)?;
2671 let dims = self.dims(input);
2672 unsafe {
2673 let mut launch = context.stream.launch_builder(&context.image.dw_fwd);
2674 launch.arg(&mut out).arg(&input.data).arg(weight.value());
2675 dims.iter().for_each(|d| {
2676 launch.arg(d);
2677 });
2678 launch
2679 .launch(crate::cuda_training::cfg(total))
2680 .map_err(crate::cuda_training::cuda_err("depthwise kernel"))?;
2681 }
2682 Ok(DeviceImageBatch {
2683 height: oh,
2684 width: ow,
2685 ..input.with_data(out)
2686 })
2687 }
2688
2689 #[cfg(feature = "cuda")]
2691 pub fn backward_device(
2692 &mut self,
2693 input: &DeviceImageBatch,
2694 grad: &DeviceImageBatch,
2695 ) -> Result<DeviceImageBatch, NetworkError> {
2696 let dims = self.dims(input);
2697 let taps = self.weight.value.cols;
2698 let Some(weight) = self.weight.device.as_mut() else {
2699 return Err(NetworkError::InvalidConfig(
2700 "backward_device needs the weight on a device".into(),
2701 ));
2702 };
2703 let context = input.context.clone();
2704 let err = crate::cuda_training::cuda_err;
2705 let total = input.data.len();
2706 let mut input_grad = context.uninit(total)?;
2707 let mut weight_grad = context.zeros(taps)?;
2708 unsafe {
2709 let mut launch = context.stream.launch_builder(&context.image.dw_bwd_input);
2710 launch
2711 .arg(&mut input_grad)
2712 .arg(&grad.data)
2713 .arg(weight.value());
2714 dims.iter().for_each(|d| {
2715 launch.arg(d);
2716 });
2717 launch
2718 .launch(crate::cuda_training::cfg(total))
2719 .map_err(err("depthwise input gradient kernel"))?;
2720 let mut launch = context.stream.launch_builder(&context.image.dw_bwd_weight);
2721 launch
2722 .arg(&mut weight_grad)
2723 .arg(&grad.data)
2724 .arg(&input.data);
2725 dims.iter().for_each(|d| {
2726 launch.arg(d);
2727 });
2728 let pixels = input.batch * dims[6] as usize * dims[7] as usize;
2730 launch
2731 .launch(crate::cuda_training::cfg(pixels.div_ceil(64) * taps))
2732 .map_err(err("depthwise weight gradient kernel"))?;
2733 }
2734 let one = context.filled(1, 1.0)?;
2735 weight.accumulate_grad_device(&one, &weight_grad, 1)?;
2736 Ok(input.with_data(input_grad))
2737 }
2738
2739 #[cfg(feature = "cuda")]
2741 fn dims(&self, input: &DeviceImageBatch) -> [i32; 8] {
2742 let (oh, ow) = self.output_size(input.height, input.width);
2743 [
2744 input.batch,
2745 input.height,
2746 input.width,
2747 input.channels,
2748 self.kernel,
2749 self.stride,
2750 oh,
2751 ow,
2752 ]
2753 .map(|v| v as i32)
2754 }
2755}
2756
2757#[derive(Clone, Copy, Debug, PartialEq, Eq)]
2761pub struct PixelAttentionShape {
2762 pub heads: usize,
2763 pub key_dim: usize,
2764 pub head_dim: usize,
2765}
2766
2767impl PixelAttentionShape {
2768 pub fn yolo(dim: usize, heads: usize) -> Self {
2770 let head_dim = dim / heads;
2771 Self {
2772 heads,
2773 key_dim: head_dim / 2,
2774 head_dim,
2775 }
2776 }
2777
2778 fn per_head(&self) -> usize {
2779 2 * self.key_dim + self.head_dim
2780 }
2781
2782 pub fn dim(&self) -> usize {
2784 self.heads * self.head_dim
2785 }
2786
2787 fn scale(&self) -> f32 {
2788 (self.key_dim as f32).powf(-0.5)
2789 }
2790
2791 fn check(&self, channels: usize) -> Result<(), NetworkError> {
2792 match channels == self.heads * self.per_head() {
2793 true => Ok(()),
2794 false => Err(NetworkError::InvalidTarget {
2795 expected: self.heads * self.per_head(),
2796 actual: channels,
2797 }),
2798 }
2799 }
2800}
2801
2802#[derive(Clone, Debug)]
2804pub struct PixelAttentionCache {
2805 qkv: ImageBatch,
2806 shape: PixelAttentionShape,
2807 probabilities: Vec<Matrix>,
2809}
2810
2811fn head_part(map: &ImageBatch, sample: usize, start: usize, width: usize) -> Matrix {
2813 let (pixels, channels) = (map.pixels(), map.channels());
2814 let mut part = Matrix::new(pixels, width);
2815 for p in 0..pixels {
2816 let row = &map.tokens.data[(sample * pixels + p) * channels..][start..start + width];
2817 part.data[p * width..(p + 1) * width].copy_from_slice(row);
2818 }
2819 part
2820}
2821
2822fn add_head_part(map: &mut ImageBatch, sample: usize, start: usize, part: &Matrix) {
2823 let (pixels, channels, width) = (map.pixels(), map.channels(), part.cols);
2824 for p in 0..pixels {
2825 map.tokens.data[(sample * pixels + p) * channels..][start..start + width]
2826 .iter_mut()
2827 .zip(part.row(p))
2828 .for_each(|(slot, v)| *slot += v);
2829 }
2830}
2831
2832pub fn pixel_attention(
2837 qkv: &ImageBatch,
2838 shape: PixelAttentionShape,
2839) -> Result<(ImageBatch, ImageBatch, PixelAttentionCache), NetworkError> {
2840 shape.check(qkv.channels())?;
2841 let (pixels, kd, hd) = (qkv.pixels(), shape.key_dim, shape.head_dim);
2842 let mut attended = qkv.like(shape.dim());
2843 let mut values = qkv.like(shape.dim());
2844 let mut probabilities = Vec::new();
2845 for b in 0..qkv.batch {
2846 for h in 0..shape.heads {
2847 let start = h * shape.per_head();
2848 let q = head_part(qkv, b, start, kd);
2849 let k = head_part(qkv, b, start + kd, kd);
2850 let v = head_part(qkv, b, start + 2 * kd, hd);
2851 let mut p = Matrix::new(pixels, pixels);
2852 q.dot_rhs_transposed(&k, &mut p);
2853 for row in p.data.chunks_exact_mut(pixels) {
2854 let max = row.iter().fold(f32::MIN, |m, &s| m.max(s * shape.scale()));
2855 let mut sum = 0.0;
2856 row.iter_mut().for_each(|s| {
2857 *s = (*s * shape.scale() - max).exp();
2858 sum += *s;
2859 });
2860 row.iter_mut().for_each(|s| *s /= sum);
2861 }
2862 let mut out = Matrix::new(pixels, hd);
2863 p.dot(&v, &mut out);
2864 add_head_part(&mut attended, b, h * hd, &out);
2865 add_head_part(&mut values, b, h * hd, &v);
2866 probabilities.push(p);
2867 }
2868 }
2869 Ok((
2870 attended,
2871 values,
2872 PixelAttentionCache {
2873 qkv: qkv.clone(),
2874 shape,
2875 probabilities,
2876 },
2877 ))
2878}
2879
2880pub fn pixel_attention_backward(
2882 cache: &PixelAttentionCache,
2883 grad_attended: &ImageBatch,
2884 grad_values: &ImageBatch,
2885) -> ImageBatch {
2886 let (qkv, shape) = (&cache.qkv, cache.shape);
2887 let (pixels, kd, hd) = (qkv.pixels(), shape.key_dim, shape.head_dim);
2888 let mut grad = qkv.like(qkv.channels());
2889 for b in 0..qkv.batch {
2890 for h in 0..shape.heads {
2891 let start = h * shape.per_head();
2892 let p = &cache.probabilities[b * shape.heads + h];
2893 let q = head_part(qkv, b, start, kd);
2894 let k = head_part(qkv, b, start + kd, kd);
2895 let v = head_part(qkv, b, start + 2 * kd, hd);
2896 let d_out = head_part(grad_attended, b, h * hd, hd);
2897 let mut d_v = head_part(grad_values, b, h * hd, hd);
2898 let mut through = Matrix::new(pixels, hd);
2899 p.dot_self_transposed(&d_out, &mut through);
2900 d_v.data
2901 .iter_mut()
2902 .zip(&through.data)
2903 .for_each(|(a, b)| *a += b);
2904 let mut d_s = Matrix::new(pixels, pixels);
2905 d_out.dot_rhs_transposed(&v, &mut d_s);
2906 for (ds, pr) in d_s
2907 .data
2908 .chunks_exact_mut(pixels)
2909 .zip(p.data.chunks_exact(pixels))
2910 {
2911 let along: f32 = ds.iter().zip(pr).map(|(d, p)| d * p).sum();
2912 ds.iter_mut()
2913 .zip(pr)
2914 .for_each(|(d, p)| *d = p * (*d - along) * shape.scale());
2915 }
2916 let (mut d_q, mut d_k) = (Matrix::new(pixels, kd), Matrix::new(pixels, kd));
2917 d_s.dot(&k, &mut d_q);
2918 d_s.dot_self_transposed(&q, &mut d_k);
2919 add_head_part(&mut grad, b, start, &d_q);
2920 add_head_part(&mut grad, b, start + kd, &d_k);
2921 add_head_part(&mut grad, b, start + 2 * kd, &d_v);
2922 }
2923 }
2924 grad
2925}
2926
2927#[cfg(feature = "cuda")]
2929pub struct DevicePixelAttentionCache {
2930 qkv: DeviceImageBatch,
2931 shape: PixelAttentionShape,
2932 probabilities: cudarc::driver::CudaSlice<f32>,
2934}
2935
2936#[cfg(feature = "cuda")]
2941pub fn pixel_attention_device(
2942 qkv: &DeviceImageBatch,
2943 shape: PixelAttentionShape,
2944) -> Result<
2945 (
2946 DeviceImageBatch,
2947 DeviceImageBatch,
2948 DevicePixelAttentionCache,
2949 ),
2950 NetworkError,
2951> {
2952 use crate::gpu_transformer::{gemm_plain_batched, gemm_rhs_transposed_batched};
2953 shape.check(qkv.channels)?;
2954 let context = &qkv.context;
2955 let (n, kd, hd, s, cq, dim) = (
2956 qkv.pixels(),
2957 shape.key_dim,
2958 shape.head_dim,
2959 shape.per_head(),
2960 qkv.channels,
2961 shape.dim(),
2962 );
2963 let (rows, span) = (qkv.batch * n, shape.heads * n * n);
2964 let mut probabilities = context.zeros(qkv.batch * span)?;
2965 let mut lse = context.zeros(qkv.batch * shape.heads * n)?;
2966 let mut attended = context.zeros(rows * dim)?;
2967 let mut values = context.zeros(rows * dim)?;
2968 for b in 0..qkv.batch {
2969 let base = b * n * cq;
2970 gemm_rhs_transposed_batched(
2971 context,
2972 &qkv.data.slice(base..),
2973 cq,
2974 &qkv.data.slice(base + kd..),
2975 cq,
2976 &mut probabilities.slice_mut(b * span..),
2977 n,
2978 n,
2979 n,
2980 kd,
2981 shape.scale(),
2982 0.0,
2983 shape.heads,
2984 s,
2985 s,
2986 n * n,
2987 )?;
2988 }
2989 let gpu = crate::gpu_model::Gpu { context };
2990 gpu.attention_softmax(
2991 &mut probabilities,
2992 &mut lse,
2993 qkv.batch * shape.heads * n,
2994 n,
2995 false,
2996 n,
2997 0,
2998 )?;
2999 for b in 0..qkv.batch {
3000 gemm_plain_batched(
3001 context,
3002 &probabilities.slice(b * span..),
3003 n,
3004 &qkv.data.slice(b * n * cq + 2 * kd..),
3005 cq,
3006 &mut attended.slice_mut(b * n * dim..),
3007 dim,
3008 n,
3009 hd,
3010 n,
3011 1.0,
3012 0.0,
3013 shape.heads,
3014 n * n,
3015 s,
3016 hd,
3017 )?;
3018 }
3019 for h in 0..shape.heads {
3020 copy_channels(
3021 context,
3022 &mut values,
3023 dim,
3024 h * hd,
3025 &qkv.data,
3026 cq,
3027 h * s + 2 * kd,
3028 hd,
3029 rows,
3030 false,
3031 )?;
3032 }
3033 Ok((
3034 DeviceImageBatch {
3035 channels: dim,
3036 ..qkv.with_data(attended)
3037 },
3038 DeviceImageBatch {
3039 channels: dim,
3040 ..qkv.with_data(values)
3041 },
3042 DevicePixelAttentionCache {
3043 qkv: qkv.clone(),
3044 shape,
3045 probabilities,
3046 },
3047 ))
3048}
3049
3050#[cfg(feature = "cuda")]
3052pub fn pixel_attention_backward_device(
3053 cache: &DevicePixelAttentionCache,
3054 grad_attended: &DeviceImageBatch,
3055 grad_values: &DeviceImageBatch,
3056) -> Result<DeviceImageBatch, NetworkError> {
3057 use crate::gpu_transformer::{
3058 gemm_lhs_transposed_batched, gemm_plain_batched, gemm_rhs_transposed_batched,
3059 };
3060 let (qkv, shape) = (&cache.qkv, cache.shape);
3061 let context = &qkv.context;
3062 let (n, kd, hd, s, cq, dim) = (
3063 qkv.pixels(),
3064 shape.key_dim,
3065 shape.head_dim,
3066 shape.per_head(),
3067 qkv.channels,
3068 shape.dim(),
3069 );
3070 let (rows, span) = (qkv.batch * n, shape.heads * n * n);
3071 let mut grad = context.zeros(rows * cq)?;
3072 for h in 0..shape.heads {
3073 copy_channels(
3074 context,
3075 &mut grad,
3076 cq,
3077 h * s + 2 * kd,
3078 &grad_values.data,
3079 dim,
3080 h * hd,
3081 hd,
3082 rows,
3083 false,
3084 )?;
3085 }
3086 let mut d_s = context.zeros(qkv.batch * span)?;
3087 for b in 0..qkv.batch {
3088 let base = b * n * cq;
3089 let d_out = grad_attended.data.slice(b * n * dim..);
3090 gemm_lhs_transposed_batched(
3092 context,
3093 &cache.probabilities.slice(b * span..),
3094 n,
3095 &d_out,
3096 dim,
3097 &mut grad.slice_mut(base + 2 * kd..),
3098 cq,
3099 n,
3100 n,
3101 hd,
3102 1.0,
3103 1.0,
3104 shape.heads,
3105 n * n,
3106 hd,
3107 s,
3108 )?;
3109 gemm_rhs_transposed_batched(
3111 context,
3112 &d_out,
3113 dim,
3114 &qkv.data.slice(base + 2 * kd..),
3115 cq,
3116 &mut d_s.slice_mut(b * span..),
3117 n,
3118 n,
3119 n,
3120 hd,
3121 1.0,
3122 0.0,
3123 shape.heads,
3124 hd,
3125 s,
3126 n * n,
3127 )?;
3128 }
3129 let gpu = crate::gpu_model::Gpu { context };
3130 gpu.softmax_backward(
3131 &mut d_s,
3132 &cache.probabilities,
3133 qkv.batch * shape.heads * n,
3134 n,
3135 false,
3136 )?;
3137 for b in 0..qkv.batch {
3138 let base = b * n * cq;
3139 gemm_plain_batched(
3141 context,
3142 &d_s.slice(b * span..),
3143 n,
3144 &qkv.data.slice(base + kd..),
3145 cq,
3146 &mut grad.slice_mut(base..),
3147 cq,
3148 n,
3149 kd,
3150 n,
3151 shape.scale(),
3152 0.0,
3153 shape.heads,
3154 n * n,
3155 s,
3156 s,
3157 )?;
3158 gemm_lhs_transposed_batched(
3159 context,
3160 &d_s.slice(b * span..),
3161 n,
3162 &qkv.data.slice(base..),
3163 cq,
3164 &mut grad.slice_mut(base + kd..),
3165 cq,
3166 n,
3167 n,
3168 kd,
3169 shape.scale(),
3170 0.0,
3171 shape.heads,
3172 n * n,
3173 s,
3174 s,
3175 )?;
3176 }
3177 Ok(qkv.with_data(grad))
3178}
3179
3180pub fn upsample_nearest_batch(input: &ImageBatch, factor: usize) -> ImageBatch {
3182 let mut output = ImageBatch::new(
3183 input.batch,
3184 input.height * factor,
3185 input.width * factor,
3186 input.channels(),
3187 );
3188 for b in 0..input.batch {
3189 for y in 0..output.height {
3190 for x in 0..output.width {
3191 let source = input.pixel(b, y / factor, x / factor).to_vec();
3192 output.pixel_mut(b, y, x).copy_from_slice(&source);
3193 }
3194 }
3195 }
3196 output
3197}
3198
3199pub fn upsample_nearest_backward(grad: &ImageBatch, factor: usize) -> ImageBatch {
3202 let mut input_grad = ImageBatch::new(
3203 grad.batch,
3204 grad.height / factor,
3205 grad.width / factor,
3206 grad.channels(),
3207 );
3208 for b in 0..grad.batch {
3209 for y in 0..grad.height {
3210 for x in 0..grad.width {
3211 let g = grad.pixel(b, y, x).to_vec();
3212 input_grad
3213 .pixel_mut(b, y / factor, x / factor)
3214 .iter_mut()
3215 .zip(g)
3216 .for_each(|(s, g)| *s += g);
3217 }
3218 }
3219 }
3220 input_grad
3221}
3222
3223pub fn concat_channels(parts: &[&ImageBatch]) -> ImageBatch {
3226 let channels = parts.iter().map(|p| p.channels()).sum();
3227 let mut output = parts[0].like(channels);
3228 let mut offset = 0;
3229 for part in parts {
3230 let width = part.channels();
3231 for (out, row) in output
3232 .tokens
3233 .data
3234 .chunks_exact_mut(channels)
3235 .zip(part.tokens.data.chunks_exact(width))
3236 {
3237 out[offset..offset + width].copy_from_slice(row);
3238 }
3239 offset += width;
3240 }
3241 output
3242}
3243
3244pub fn split_channels(input: &ImageBatch, widths: &[usize]) -> Vec<ImageBatch> {
3247 let mut offset = 0;
3248 widths
3249 .iter()
3250 .map(|&width| {
3251 let mut part = input.like(width);
3252 for (out, row) in part
3253 .tokens
3254 .data
3255 .chunks_exact_mut(width)
3256 .zip(input.tokens.data.chunks_exact(input.channels()))
3257 {
3258 out.copy_from_slice(&row[offset..offset + width]);
3259 }
3260 offset += width;
3261 part
3262 })
3263 .collect()
3264}
3265
3266#[derive(Clone, Debug)]
3268pub struct MaxPoolCache {
3269 kernel: usize,
3270 argmax: Vec<u32>,
3271}
3272
3273pub fn max_pool_same(input: &ImageBatch, kernel: usize) -> (ImageBatch, MaxPoolCache) {
3277 let (height, width, channels, pad) = (input.height, input.width, input.channels(), kernel / 2);
3278 let mut output = input.like(channels);
3279 let mut argmax = vec![0u32; input.tokens.data.len()];
3280 for b in 0..input.batch {
3281 for y in 0..height {
3282 for x in 0..width {
3283 let at = ((b * height + y) * width + x) * channels;
3284 for c in 0..channels {
3285 let (mut best, mut arg) = (f32::MIN, 0);
3286 for ky in 0..kernel {
3287 for kx in 0..kernel {
3288 let (sy, sx) = ((y + ky).wrapping_sub(pad), (x + kx).wrapping_sub(pad));
3289 if sy < height && sx < width {
3290 let v = input.pixel(b, sy, sx)[c];
3291 if v > best {
3292 (best, arg) = (v, ky * kernel + kx);
3293 }
3294 }
3295 }
3296 }
3297 output.tokens.data[at + c] = best;
3298 argmax[at + c] = arg as u32;
3299 }
3300 }
3301 }
3302 }
3303 (output, MaxPoolCache { kernel, argmax })
3304}
3305
3306pub fn max_pool_same_backward(cache: &MaxPoolCache, grad: &ImageBatch) -> ImageBatch {
3308 let (height, width, channels, kernel) =
3309 (grad.height, grad.width, grad.channels(), cache.kernel);
3310 let pad = kernel / 2;
3311 let mut input_grad = grad.like(channels);
3312 for (index, (&g, &arg)) in grad.tokens.data.iter().zip(&cache.argmax).enumerate() {
3313 let (c, pixel) = (index % channels, index / channels);
3314 let (x, y, b) = (
3315 pixel % width,
3316 pixel / width % height,
3317 pixel / width / height,
3318 );
3319 let (ky, kx) = (arg as usize / kernel, arg as usize % kernel);
3320 let (sy, sx) = (y + ky - pad, x + kx - pad);
3321 input_grad.tokens.data[((b * height + sy) * width + sx) * channels + c] += g;
3322 }
3323 input_grad
3324}
3325
3326#[cfg(feature = "cuda")]
3328pub fn upsample_nearest_device(
3329 input: &DeviceImageBatch,
3330 factor: usize,
3331) -> Result<DeviceImageBatch, NetworkError> {
3332 let total = input.data.len() * factor * factor;
3333 let out = image_launch(
3334 input,
3335 total,
3336 "upsample kernel",
3337 |context| &context.image.upsample,
3338 &[input.height, input.width, input.channels, factor],
3339 )?;
3340 Ok(DeviceImageBatch {
3341 height: input.height * factor,
3342 width: input.width * factor,
3343 ..input.with_data(out)
3344 })
3345}
3346
3347#[cfg(feature = "cuda")]
3349pub fn upsample_nearest_backward_device(
3350 grad: &DeviceImageBatch,
3351 factor: usize,
3352) -> Result<DeviceImageBatch, NetworkError> {
3353 let (height, width) = (grad.height / factor, grad.width / factor);
3354 let total = grad.data.len() / (factor * factor);
3355 let out = image_launch(
3356 grad,
3357 total,
3358 "upsample backward kernel",
3359 |context| &context.image.upsample_bwd,
3360 &[height, width, grad.channels, factor],
3361 )?;
3362 Ok(DeviceImageBatch {
3363 height,
3364 width,
3365 ..grad.with_data(out)
3366 })
3367}
3368
3369#[cfg(feature = "cuda")]
3371pub fn add_device(
3372 a: &DeviceImageBatch,
3373 b: &DeviceImageBatch,
3374) -> Result<DeviceImageBatch, NetworkError> {
3375 let context = &a.context;
3376 let total = a.data.len();
3377 if b.data.len() != total {
3378 return Err(NetworkError::InvalidTarget {
3379 expected: total,
3380 actual: b.data.len(),
3381 });
3382 }
3383 let mut out = context.uninit(total)?;
3384 let n = total as i32;
3385 unsafe {
3386 context
3387 .stream
3388 .launch_builder(&context.image.add)
3389 .arg(&mut out)
3390 .arg(&a.data)
3391 .arg(&b.data)
3392 .arg(&n)
3393 .launch(crate::cuda_training::cfg(total))
3394 .map_err(crate::cuda_training::cuda_err("add kernel"))?;
3395 }
3396 Ok(a.with_data(out))
3397}
3398
3399#[cfg(feature = "cuda")]
3401pub fn concat_channels_device(
3402 parts: &[&DeviceImageBatch],
3403) -> Result<DeviceImageBatch, NetworkError> {
3404 let first = parts[0];
3405 let channels: usize = parts.iter().map(|p| p.channels).sum();
3406 let rows = first.batch * first.pixels();
3407 let mut out = first.context.uninit(rows * channels)?;
3408 let mut offset = 0;
3409 for part in parts {
3410 copy_channels(
3411 &first.context,
3412 &mut out,
3413 channels,
3414 offset,
3415 &part.data,
3416 part.channels,
3417 0,
3418 part.channels,
3419 rows,
3420 false,
3421 )?;
3422 offset += part.channels;
3423 }
3424 Ok(DeviceImageBatch {
3425 channels,
3426 ..first.with_data(out)
3427 })
3428}
3429
3430#[cfg(feature = "cuda")]
3432pub fn split_channels_device(
3433 input: &DeviceImageBatch,
3434 widths: &[usize],
3435) -> Result<Vec<DeviceImageBatch>, NetworkError> {
3436 let mut offset = 0;
3437 widths
3438 .iter()
3439 .map(|&width| {
3440 offset += width;
3441 slice_channels_device(input, offset - width, width)
3442 })
3443 .collect()
3444}
3445
3446#[cfg(feature = "cuda")]
3448pub(crate) fn add_channels_device(
3449 dst: &mut DeviceImageBatch,
3450 offset: usize,
3451 src: &DeviceImageBatch,
3452) -> Result<(), NetworkError> {
3453 let rows = src.batch * src.pixels();
3454 let context = dst.context.clone();
3455 copy_channels(
3456 &context,
3457 &mut dst.data,
3458 dst.channels,
3459 offset,
3460 &src.data,
3461 src.channels,
3462 0,
3463 src.channels,
3464 rows,
3465 true,
3466 )
3467}
3468
3469#[cfg(feature = "cuda")]
3471pub fn slice_channels_device(
3472 input: &DeviceImageBatch,
3473 offset: usize,
3474 width: usize,
3475) -> Result<DeviceImageBatch, NetworkError> {
3476 let rows = input.batch * input.pixels();
3477 let mut out = input.context.uninit(rows * width)?;
3478 copy_channels(
3479 &input.context,
3480 &mut out,
3481 width,
3482 0,
3483 &input.data,
3484 input.channels,
3485 offset,
3486 width,
3487 rows,
3488 false,
3489 )?;
3490 Ok(DeviceImageBatch {
3491 channels: width,
3492 ..input.with_data(out)
3493 })
3494}
3495
3496#[cfg(feature = "cuda")]
3497#[allow(clippy::too_many_arguments)]
3498fn copy_channels(
3499 context: &crate::gpu_transformer::GpuContext,
3500 dst: &mut cudarc::driver::CudaSlice<f32>,
3501 dst_channels: usize,
3502 dst_offset: usize,
3503 src: &cudarc::driver::CudaSlice<f32>,
3504 src_channels: usize,
3505 src_offset: usize,
3506 width: usize,
3507 rows: usize,
3508 add: bool,
3509) -> Result<(), NetworkError> {
3510 let [rows_arg, width_arg, dc, doff, sc, soff, add] = [
3511 rows,
3512 width,
3513 dst_channels,
3514 dst_offset,
3515 src_channels,
3516 src_offset,
3517 add as usize,
3518 ]
3519 .map(|v| v as i32);
3520 unsafe {
3521 context
3522 .stream
3523 .launch_builder(&context.image.copy_channels)
3524 .arg(dst)
3525 .arg(src)
3526 .arg(&rows_arg)
3527 .arg(&width_arg)
3528 .arg(&dc)
3529 .arg(&doff)
3530 .arg(&sc)
3531 .arg(&soff)
3532 .arg(&add)
3533 .launch(crate::cuda_training::cfg(rows * width))
3534 .map_err(crate::cuda_training::cuda_err("channel copy kernel"))?;
3535 }
3536 Ok(())
3537}
3538
3539#[cfg(feature = "cuda")]
3541pub struct DeviceMaxPoolCache {
3542 kernel: usize,
3543 argmax: cudarc::driver::CudaSlice<i32>,
3544}
3545
3546#[cfg(feature = "cuda")]
3548pub fn max_pool_same_device(
3549 input: &DeviceImageBatch,
3550 kernel: usize,
3551) -> Result<(DeviceImageBatch, DeviceMaxPoolCache), NetworkError> {
3552 let context = &input.context;
3553 let total = input.data.len();
3554 let mut out = context.uninit(total)?;
3555 let mut argmax = context
3556 .stream
3557 .alloc_zeros::<i32>(total)
3558 .map_err(crate::cuda_training::cuda_err("max pool allocation"))?;
3559 let [n, h, w, c, k] =
3560 [total, input.height, input.width, input.channels, kernel].map(|v| v as i32);
3561 unsafe {
3562 context
3563 .stream
3564 .launch_builder(&context.image.maxpool)
3565 .arg(&mut out)
3566 .arg(&mut argmax)
3567 .arg(&input.data)
3568 .arg(&n)
3569 .arg(&h)
3570 .arg(&w)
3571 .arg(&c)
3572 .arg(&k)
3573 .launch(crate::cuda_training::cfg(total))
3574 .map_err(crate::cuda_training::cuda_err("max pool kernel"))?;
3575 }
3576 Ok((input.with_data(out), DeviceMaxPoolCache { kernel, argmax }))
3577}
3578
3579#[cfg(feature = "cuda")]
3582pub fn max_pool_same_backward_device(
3583 cache: &DeviceMaxPoolCache,
3584 grad: &DeviceImageBatch,
3585) -> Result<DeviceImageBatch, NetworkError> {
3586 let context = &grad.context;
3587 let total = grad.data.len();
3588 let mut out = context.zeros(total)?;
3589 let [n, h, w, c, k] =
3590 [total, grad.height, grad.width, grad.channels, cache.kernel].map(|v| v as i32);
3591 unsafe {
3592 context
3593 .stream
3594 .launch_builder(&context.image.maxpool_bwd)
3595 .arg(&mut out)
3596 .arg(&grad.data)
3597 .arg(&cache.argmax)
3598 .arg(&n)
3599 .arg(&h)
3600 .arg(&w)
3601 .arg(&c)
3602 .arg(&k)
3603 .launch(crate::cuda_training::cfg(total))
3604 .map_err(crate::cuda_training::cuda_err("max pool backward kernel"))?;
3605 }
3606 Ok(grad.with_data(out))
3607}
3608
3609#[cfg(feature = "cuda")]
3611fn image_launch(
3612 input: &DeviceImageBatch,
3613 total: usize,
3614 what: &'static str,
3615 kernel: impl Fn(&crate::gpu_transformer::GpuContext) -> &cudarc::driver::CudaFunction,
3616 dims: &[usize],
3617) -> Result<cudarc::driver::CudaSlice<f32>, NetworkError> {
3618 let context = &input.context;
3619 let mut out = context.uninit(total)?;
3620 let n = total as i32;
3621 let dims: Vec<i32> = dims.iter().map(|&d| d as i32).collect();
3622 let mut launch = context.stream.launch_builder(kernel(context));
3623 launch.arg(&mut out).arg(&input.data).arg(&n);
3624 for d in &dims {
3625 launch.arg(d);
3626 }
3627 unsafe { launch.launch(crate::cuda_training::cfg(total)) }
3628 .map_err(crate::cuda_training::cuda_err(what))?;
3629 Ok(out)
3630}
3631
3632#[cfg(test)]
3633mod tests {
3634 use super::*;
3635 use crate::optimizers::Optimizer;
3636 use rand::SeedableRng;
3637
3638 fn noise(
3640 batch: usize,
3641 height: usize,
3642 width: usize,
3643 channels: usize,
3644 seed: usize,
3645 ) -> ImageBatch {
3646 let data = (0..batch * height * width * channels)
3647 .map(|index| (((index * 37 + seed * 11) % 197) as f32 / 98.0) - 1.0)
3648 .collect();
3649 ImageBatch::from_tokens(
3650 batch,
3651 height,
3652 width,
3653 Matrix::from_vec(batch * height * width, channels, data),
3654 )
3655 }
3656
3657 fn mse(prediction: &ImageBatch, target: &ImageBatch) -> (f32, ImageBatch) {
3659 let count = prediction.tokens.data.len() as f32;
3660 let mut grad = prediction.like(prediction.channels());
3661 let mut loss = 0.0;
3662 for (index, (value, wanted)) in prediction
3663 .tokens
3664 .data
3665 .iter()
3666 .zip(&target.tokens.data)
3667 .enumerate()
3668 {
3669 let difference = value - wanted;
3670 loss += difference * difference / count;
3671 grad.tokens.data[index] = 2.0 * difference / count;
3672 }
3673 (loss, grad)
3674 }
3675
3676 fn loss_curve(
3678 conv: &mut TrainableConv2d,
3679 input: &ImageBatch,
3680 target: &ImageBatch,
3681 steps: usize,
3682 ) -> Result<Vec<f32>, NetworkError> {
3683 let optimizer = Optimizer::adam(1e-2);
3684 let mut curve = Vec::with_capacity(steps);
3685 for step in 1..=steps {
3686 let (prediction, cache) = conv.forward(input)?;
3687 let (loss, grad_output) = mse(&prediction, target);
3688 curve.push(loss);
3689 for param in conv.params_mut() {
3690 param.zero_grad();
3691 }
3692 conv.backward(&cache, &grad_output)?;
3693 for param in conv.params_mut() {
3694 param.step(&optimizer, step, 1.0);
3695 }
3696 }
3697 Ok(curve)
3698 }
3699
3700 fn pooled_gap(
3703 f: impl Fn(&ImageBatch) -> Matrix,
3704 input: &ImageBatch,
3705 weights: &[f32],
3706 analytic: &[f32],
3707 ) -> f32 {
3708 let loss = |input: &ImageBatch| -> f32 {
3709 f(input).data.iter().zip(weights).map(|(a, b)| a * b).sum()
3710 };
3711 let scale = analytic.iter().fold(0f32, |m, g| m.max(g.abs()));
3712 let mut moved = input.clone();
3713 let mut worst = 0f32;
3714 for (index, analytic) in analytic.iter().enumerate() {
3715 let h = 1e-2;
3716 moved.tokens.data[index] += h;
3717 let up = loss(&moved);
3718 moved.tokens.data[index] -= 2.0 * h;
3719 let down = loss(&moved);
3720 moved.tokens.data[index] += h;
3721 worst = worst.max(((up - down) / (2.0 * h) - analytic).abs() / scale);
3722 }
3723 worst
3724 }
3725
3726 #[test]
3727 fn spatial_softmax_matches_finite_differences_and_finds_a_peak() {
3728 let mut input = noise(2, 3, 4, 5, 3);
3729 input.tokens.data.iter_mut().for_each(|v| *v *= 3.0);
3730 let weights = noise(1, 1, 20, 1, 4).tokens.data;
3731 let (_, cache) = spatial_softmax(&input);
3732 let grad = spatial_softmax_backward(&cache, &Matrix::from_vec(2, 10, weights.clone()));
3733 let gap = pooled_gap(
3734 |x| spatial_softmax(x).0,
3735 &input,
3736 &weights,
3737 &grad.tokens.data,
3738 );
3739 assert!(gap < 1e-2, "{gap}");
3740
3741 let mut spike = ImageBatch::new(2, 3, 4, 5);
3743 spike.tokens.data[3 * 5 + 2] = 50.0;
3744 let (expected, _) = spatial_softmax(&spike);
3745 assert!((expected.data[2] - 1.0).abs() < 1e-4);
3746 assert!((expected.data[5 + 2] + 1.0).abs() < 1e-4);
3747 }
3748
3749 #[test]
3750 fn trainable_group_norm_matches_the_inference_one_and_finite_differences() {
3751 let input = noise(2, 3, 4, 8, 3);
3752 let mut norm = TrainableGroupNorm::new(4, 8, 1e-5).unwrap();
3753 norm.gamma.value.data = noise(1, 1, 8, 1, 4)
3754 .tokens
3755 .data
3756 .iter()
3757 .map(|v| v + 1.5)
3758 .collect();
3759 norm.beta.value.data = noise(1, 1, 8, 1, 5).tokens.data;
3760 let fixed = GroupNorm::new(
3761 4,
3762 norm.gamma.value.data.clone(),
3763 norm.beta.value.data.clone(),
3764 1e-5,
3765 )
3766 .unwrap();
3767 let (output, cache) = norm.forward(&input).unwrap();
3768 for sample in 0..2 {
3769 let rows = 12 * sample..12 * (sample + 1);
3770 let tokens = Matrix::from_vec(
3771 12,
3772 8,
3773 input.tokens.data[rows.start * 8..rows.end * 8].to_vec(),
3774 );
3775 let mut map = FeatureMap::from_tokens(&tokens, 3, 4).unwrap();
3776 fixed.forward(&mut map).unwrap();
3777 assert_eq!(
3778 map.to_tokens().data,
3779 output.tokens.data[rows.start * 8..rows.end * 8]
3780 );
3781 }
3782
3783 let weights = noise(2, 3, 4, 8, 6);
3784 let grad = norm.backward(&cache, &weights);
3785 let through = |input: &ImageBatch| norm.forward(input).unwrap().0.tokens;
3786 let gap = pooled_gap(through, &input, &weights.tokens.data, &grad.tokens.data);
3787 assert!(gap < 1e-3, "input {gap}");
3788 for (which, analytic) in [(0, &norm.gamma.grad.data), (1, &norm.beta.grad.data)] {
3789 let params = ImageBatch::from_tokens(
3790 1,
3791 1,
3792 8,
3793 Matrix::from_vec(
3794 8,
3795 1,
3796 if which == 0 {
3797 norm.gamma.value.data.clone()
3798 } else {
3799 norm.beta.value.data.clone()
3800 },
3801 ),
3802 );
3803 let through = |params: &ImageBatch| {
3804 let mut moved = norm.clone();
3805 let target = if which == 0 {
3806 &mut moved.gamma
3807 } else {
3808 &mut moved.beta
3809 };
3810 target.value.data.copy_from_slice(¶ms.tokens.data);
3811 moved.forward(&input).unwrap().0.tokens
3812 };
3813 let gap = pooled_gap(through, ¶ms, &weights.tokens.data, analytic);
3814 assert!(gap < 1e-3, "param {which}: {gap}");
3815 }
3816 assert!(TrainableGroupNorm::new(3, 8, 1e-5).is_err());
3817 }
3818
3819 #[test]
3820 fn global_avg_pool_matches_finite_differences() {
3821 let input = noise(2, 3, 4, 5, 5);
3822 let weights = noise(1, 1, 10, 1, 6).tokens.data;
3823 let grad = global_avg_pool_backward(&Matrix::from_vec(2, 5, weights.clone()), 3, 4);
3824 let gap = pooled_gap(global_avg_pool, &input, &weights, &grad.tokens.data);
3825 assert!(gap < 1e-2, "{gap}");
3826 assert!((global_avg_pool(&ImageBatch::new(1, 2, 2, 3)).data[0]).abs() < 1e-9);
3827 }
3828
3829 #[test]
3830 fn a_convolution_without_bias_adds_and_trains_none() {
3831 let mut rng = rand::rngs::StdRng::seed_from_u64(12);
3832 let mut biased = TrainableConv2d::new(3, 4, 3, 2, &mut rng).unwrap();
3833 biased.bias.value.data.fill(1.0);
3834 let mut plain = biased.clone().without_bias();
3835 assert!(!plain.has_bias());
3836 assert_eq!(plain.params_mut().len() + 1, biased.params_mut().len());
3837
3838 let (input, target) = (noise(2, 5, 6, 3, 1), noise(2, 3, 3, 4, 2));
3839 let (with, with_cache) = biased.forward(&input).unwrap();
3840 let (without, cache) = plain.forward(&input).unwrap();
3841 for (with, without) in with.tokens.data.iter().zip(&without.tokens.data) {
3842 assert!((with - without - 1.0).abs() < 1e-6);
3843 }
3844 let (_, grad) = mse(&without, &target);
3845 biased.backward(&with_cache, &grad).unwrap();
3846 plain.backward(&cache, &grad).unwrap();
3847 assert_eq!(
3848 plain.weight.weight.grad.data,
3849 biased.weight.weight.grad.data
3850 );
3851 assert!(plain.bias.grad.data.iter().all(|g| *g == 0.0));
3852 assert!(plain.bias.value.data.iter().all(|b| *b == 0.0));
3853 }
3854
3855 #[test]
3856 fn a_trainable_convolution_matches_finite_differences() {
3857 let mut rng = rand::rngs::StdRng::seed_from_u64(11);
3858 let mut conv = TrainableConv2d::new(3, 4, 3, 1, &mut rng).unwrap();
3859 let input = noise(2, 5, 6, 3, 1);
3860 let target = noise(2, 5, 6, 4, 2);
3861
3862 let (prediction, cache) = conv.forward(&input).unwrap();
3863 let (_, grad_output) = mse(&prediction, &target);
3864 let grad_input = conv.backward(&cache, &grad_output).unwrap();
3865
3866 let epsilon = 1e-3;
3867 for index in [0usize, 17, 43, 88] {
3868 let mut moved = input.clone();
3869 moved.tokens.data[index] += epsilon;
3870 let (up, _) = conv.forward(&moved).unwrap();
3871 moved.tokens.data[index] -= 2.0 * epsilon;
3872 let (down, _) = conv.forward(&moved).unwrap();
3873 let numerical = (mse(&up, &target).0 - mse(&down, &target).0) / (2.0 * epsilon);
3874 let analytic = grad_input.tokens.data[index];
3875 assert!(
3876 (numerical - analytic).abs() < 1e-3,
3877 "dL/dinput[{index}]: {analytic} against {numerical} by finite difference"
3878 );
3879 }
3880
3881 for index in [0usize, 7, 26] {
3882 let mut moved = conv.clone();
3883 moved.weight.weight.value.data[index] += epsilon;
3884 let up = mse(&moved.forward(&input).unwrap().0, &target).0;
3885 moved.weight.weight.value.data[index] -= 2.0 * epsilon;
3886 let down = mse(&moved.forward(&input).unwrap().0, &target).0;
3887 let numerical = (up - down) / (2.0 * epsilon);
3888 let analytic = conv.weight.weight.grad.data[index];
3889 assert!(
3890 (numerical - analytic).abs() < 1e-3,
3891 "dL/dweight[{index}]: {analytic} against {numerical} by finite difference"
3892 );
3893 }
3894 }
3895
3896 #[test]
3897 fn training_a_convolution_drives_its_loss_down() {
3898 let mut rng = rand::rngs::StdRng::seed_from_u64(17);
3899 let mut conv = TrainableConv2d::new(8, 8, 3, 1, &mut rng).unwrap();
3900 let input = noise(1, 8, 8, 8, 9);
3901 let target = noise(1, 8, 8, 8, 10);
3902 let curve = loss_curve(&mut conv, &input, &target, 20).unwrap();
3903 assert!(
3904 curve[19] < curve[0] / 4.0,
3905 "twenty Adam steps went from {} to {}",
3906 curve[0],
3907 curve[19]
3908 );
3909 }
3910
3911 #[test]
3912 fn a_strided_convolution_shrinks_the_map_and_still_reaches_every_pixel() {
3913 let mut rng = rand::rngs::StdRng::seed_from_u64(5);
3914 let mut conv = TrainableConv2d::new(2, 3, 3, 2, &mut rng).unwrap();
3915 let input = noise(1, 8, 8, 2, 3);
3916 let (output, cache) = conv.forward(&input).unwrap();
3917 assert_eq!((output.height, output.width), (4, 4));
3918
3919 let grad_output = noise(1, 4, 4, 3, 4);
3920 let grad_input = conv.backward(&cache, &grad_output).unwrap();
3921 assert_eq!(grad_input.tokens.rows, input.tokens.rows);
3922 assert!(
3923 grad_input.tokens.data.iter().any(|value| *value != 0.0),
3924 "a stride of two still reads most of the input"
3925 );
3926 }
3927
3928 #[test]
3929 fn rebuilding_the_columns_reaches_the_same_gradients_as_keeping_them() {
3930 let mut rng = rand::rngs::StdRng::seed_from_u64(31);
3931 let mut keeping = TrainableConv2d::new(6, 8, 3, 2, &mut rng).unwrap();
3932 let mut rebuilding = keeping.clone();
3933 rebuilding.set_recompute_columns(true);
3934 let input = noise(2, 11, 9, 6, 3);
3935
3936 let (kept, kept_cache) = keeping.forward(&input).unwrap();
3937 let (rebuilt, rebuilt_cache) = rebuilding.forward(&input).unwrap();
3938 assert_eq!(
3939 kept.tokens.data, rebuilt.tokens.data,
3940 "the forward pass is the same either way"
3941 );
3942 assert!(kept_cache.holds_columns());
3943 assert!(
3944 !rebuilt_cache.holds_columns(),
3945 "the whole point is that the columns are not held"
3946 );
3947
3948 let grad_output = noise(2, kept.height, kept.width, 8, 4);
3949 let kept_input = keeping.backward(&kept_cache, &grad_output).unwrap();
3950 let rebuilt_input = rebuilding.backward(&rebuilt_cache, &grad_output).unwrap();
3951
3952 assert_eq!(kept_input.tokens.data, rebuilt_input.tokens.data);
3953 assert_eq!(
3954 keeping.weight.weight.grad.data,
3955 rebuilding.weight.weight.grad.data
3956 );
3957 assert_eq!(keeping.bias.grad.data, rebuilding.bias.grad.data);
3958 }
3959
3960 #[test]
3963 fn a_convolution_that_rebuilds_its_columns_trains_to_the_same_place() {
3964 let mut rng = rand::rngs::StdRng::seed_from_u64(37);
3965 let mut keeping = TrainableConv2d::new(8, 8, 3, 1, &mut rng).unwrap();
3966 let mut rebuilding = keeping.clone();
3967 rebuilding.set_recompute_columns(true);
3968 let input = noise(1, 12, 12, 8, 5);
3969 let target = noise(1, 12, 12, 8, 6);
3970
3971 let kept = loss_curve(&mut keeping, &input, &target, 40).unwrap();
3972 let rebuilt = loss_curve(&mut rebuilding, &input, &target, 40).unwrap();
3973 assert!(
3974 kept[39] < kept[0] / 2.0,
3975 "the reference run has to be learning"
3976 );
3977 assert_eq!(kept, rebuilt);
3978 }
3979
3980 #[cfg(feature = "cuda")]
3981 fn cuda_or_skip() -> Option<crate::param::CudaDevice> {
3982 match crate::cuda_training::cuda_doctor(0, 4096) {
3983 Ok(_) => Some(crate::param::CudaDevice::new(0, 4096).expect("the CUDA doctor passed")),
3984 Err(NetworkError::Cuda(message))
3985 if message.contains("NO_DEVICE") || message.contains("no CUDA-capable device") =>
3986 {
3987 crate::cuda_training::skip_without_cuda();
3988 None
3989 }
3990 Err(error) => panic!("CUDA is present but the CUDA doctor failed: {error}"),
3991 }
3992 }
3993
3994 #[cfg(feature = "cuda")]
3998 #[test]
3999 fn a_convolution_trained_on_the_device_follows_the_hosts_loss_curve() {
4000 let Some(device) = cuda_or_skip() else {
4001 return;
4002 };
4003 let mut rng = rand::rngs::StdRng::seed_from_u64(29);
4004 let mut host = TrainableConv2d::new(96, 96, 3, 1, &mut rng).unwrap();
4005 let mut resident = host.clone();
4006 let input = noise(1, 64, 64, 96, 7);
4007 let target = noise(1, 64, 64, 96, 8);
4008
4009 let on_host = loss_curve(&mut host, &input, &target, 100).unwrap();
4010 resident.to_cuda_on(&device).unwrap();
4011 let on_device = loss_curve(&mut resident, &input, &target, 100).unwrap();
4012
4013 assert!(
4014 on_host[99] < on_host[0] / 2.0,
4015 "the host path has to be learning"
4016 );
4017 for (step, (a, b)) in on_host.iter().zip(&on_device).enumerate() {
4018 assert!(
4019 (a - b).abs() <= 1e-4 + 1e-3 * a.abs(),
4020 "step {step}: {a} on the host against {b} on the device"
4021 );
4022 }
4023
4024 resident.to_cpu().unwrap();
4026 for (index, (a, b)) in host
4027 .weight
4028 .weight
4029 .value
4030 .data
4031 .iter()
4032 .zip(&resident.weight.weight.value.data)
4033 .enumerate()
4034 {
4035 assert!(
4036 (a - b).abs() < 1e-3,
4037 "weight[{index}]: {a} on the host against {b} on the device"
4038 );
4039 }
4040 }
4041
4042 #[cfg(feature = "cuda")]
4046 #[test]
4047 fn a_mixed_precision_convolution_tracks_the_host() {
4048 let Some(_) = cuda_or_skip() else {
4049 return;
4050 };
4051 let device = crate::param::CudaDevice::with_mixed_precision(0, 1024).unwrap();
4052 for (channels, recompute) in [(8, false), (6, true)] {
4054 let mut rng = rand::rngs::StdRng::seed_from_u64(32);
4055 let mut conv = TrainableConv2d::new(channels, 16, 3, 1, &mut rng).unwrap();
4056 conv.bias.value.data = noise(1, 1, 16, 1, 3).tokens.data;
4057 conv.set_recompute_columns(recompute);
4058 let mut host = conv.clone();
4059 conv.to_cuda_on(&device).unwrap();
4060 let input = noise(2, 10, 9, channels, 1);
4061 let grad = noise(2, 10, 9, 16, 2);
4062 let (expected, cache) = host.forward(&input).unwrap();
4063 let expected_grad = host.backward(&cache, &grad).unwrap();
4064 let (output, cache) = conv
4065 .forward_device(&device.upload(&input).unwrap())
4066 .unwrap();
4067 let input_grad = conv
4068 .backward_device(&cache, &device.upload(&grad).unwrap())
4069 .unwrap();
4070 let near = |a: &[f32], b: &[f32], what: &str| {
4072 let gap: f32 = a.iter().zip(b).map(|(a, b)| (a - b).powi(2)).sum();
4073 let scale: f32 = b.iter().map(|b| b * b).sum();
4074 assert!(
4075 gap.sqrt() < 1e-2 * scale.sqrt(),
4076 "{what}: {gap} against {scale}"
4077 );
4078 };
4079 near(
4080 &output.download().unwrap().tokens.data,
4081 &expected.tokens.data,
4082 "output",
4083 );
4084 near(
4085 &input_grad.download().unwrap().tokens.data,
4086 &expected_grad.tokens.data,
4087 "input grad",
4088 );
4089 conv.to_cpu().unwrap();
4090 near(
4091 &conv.weight.weight.grad.data,
4092 &host.weight.weight.grad.data,
4093 "weight grad",
4094 );
4095 near(&conv.bias.grad.data, &host.bias.grad.data, "bias grad");
4096 }
4097 }
4098
4099 #[cfg(feature = "cuda")]
4103 #[test]
4104 fn a_device_chain_stays_on_the_card_and_matches_the_host() {
4105 let Some(device) = cuda_or_skip() else {
4106 return;
4107 };
4108 let mut rng = rand::rngs::StdRng::seed_from_u64(31);
4109 let mut first = TrainableConv2d::new(6, 8, 3, 2, &mut rng).unwrap();
4110 let mut second = TrainableConv2d::new(8, 5, 1, 1, &mut rng).unwrap();
4112 first.bias.value.data = noise(1, 1, 8, 1, 3).tokens.data;
4113 second.bias.value.data = noise(1, 1, 5, 1, 4).tokens.data;
4114 second.set_recompute_columns(true);
4115 let (mut host_first, mut host_second) = (first.clone(), second.clone());
4116 first.to_cuda_on(&device).unwrap();
4117 second.to_cuda_on(&device).unwrap();
4118
4119 let input = noise(3, 11, 9, 6, 1);
4120 let (middle, first_cache) = host_first.forward(&input).unwrap();
4121 let (expected, second_cache) = host_second.forward(&middle).unwrap();
4122 let grad = noise(3, expected.height, expected.width, 5, 2);
4123 let expected_grad = host_first
4124 .backward(
4125 &first_cache,
4126 &host_second.backward(&second_cache, &grad).unwrap(),
4127 )
4128 .unwrap();
4129
4130 let context = device.context();
4131 let resident = device.upload(&input).unwrap();
4132 let resident_grad = device.upload(&grad).unwrap();
4133 let before = context.transfers();
4134 let (middle, first_cache) = first.forward_device(&resident).unwrap();
4135 let (output, second_cache) = second.forward_device(&middle).unwrap();
4136 let back = second
4137 .backward_device(&second_cache, &resident_grad)
4138 .unwrap();
4139 let input_grad = first.backward_device(&first_cache, &back).unwrap();
4140 assert_eq!(
4141 context.transfers(),
4142 before,
4143 "nothing crosses inside the chain"
4144 );
4145
4146 let close = |a: &[f32], b: &[f32], what: &str| {
4147 assert_eq!(a.len(), b.len(), "{what}");
4148 for (index, (a, b)) in a.iter().zip(b).enumerate() {
4149 assert!(
4150 (a - b).abs() < 1e-4 + 1e-4 * b.abs(),
4151 "{what}[{index}]: {a} against {b}"
4152 );
4153 }
4154 };
4155 close(
4156 &device.download(&output).unwrap().tokens.data,
4157 &expected.tokens.data,
4158 "output",
4159 );
4160 close(
4161 &input_grad.download().unwrap().tokens.data,
4162 &expected_grad.tokens.data,
4163 "input grad",
4164 );
4165 first.to_cpu().unwrap();
4166 second.to_cpu().unwrap();
4167 for (device_side, host_side, what) in [
4168 (&first, &host_first, "first"),
4169 (&second, &host_second, "second"),
4170 ] {
4171 close(
4172 &device_side.weight.weight.grad.data,
4173 &host_side.weight.weight.grad.data,
4174 what,
4175 );
4176 close(&device_side.bias.grad.data, &host_side.bias.grad.data, what);
4177 }
4178 }
4179
4180 #[cfg(feature = "cuda")]
4183 #[test]
4184 fn device_group_norm_and_silu_match_the_host() {
4185 let Some(device) = cuda_or_skip() else {
4186 return;
4187 };
4188 let input = noise(2, 16, 17, 12, 1);
4189 let grad = noise(2, 16, 17, 12, 2);
4190 let mut host = TrainableGroupNorm::new(4, 12, 1e-5).unwrap();
4191 host.gamma.value.data = noise(1, 1, 12, 1, 3)
4192 .tokens
4193 .data
4194 .iter()
4195 .map(|v| v + 1.2)
4196 .collect();
4197 host.beta.value.data = noise(1, 1, 12, 1, 4).tokens.data;
4198 let mut resident = host.clone();
4199 resident.to_cuda_on(&device).unwrap();
4200
4201 let (normed, cache) = host.forward(&input).unwrap();
4202 let mut activated = normed.clone();
4203 let mut back = grad.clone();
4204 for ((value, grad), pre) in activated
4205 .tokens
4206 .data
4207 .iter_mut()
4208 .zip(&mut back.tokens.data)
4209 .zip(&normed.tokens.data)
4210 {
4211 *value = crate::ffn::silu(*pre);
4212 *grad *= crate::ffn::silu_derivative(*pre);
4213 }
4214 let expected_grad = host.backward(&cache, &back);
4215
4216 let context = device.context();
4217 let (on_card, on_card_grad) = (
4218 device.upload(&input).unwrap(),
4219 device.upload(&grad).unwrap(),
4220 );
4221 let before = context.transfers();
4222 let (normed, cache) = resident.forward_device(&on_card).unwrap();
4223 let output = silu_device(&normed).unwrap();
4224 let back = silu_backward_device(&normed, &on_card_grad).unwrap();
4225 let input_grad = resident.backward_device(&cache, &back).unwrap();
4226 assert_eq!(
4227 context.transfers(),
4228 before,
4229 "nothing crosses inside the chain"
4230 );
4231
4232 let close = |a: &[f32], b: &[f32], what: &str| {
4233 assert_eq!(a.len(), b.len(), "{what}");
4234 for (index, (a, b)) in a.iter().zip(b).enumerate() {
4235 assert!(
4236 (a - b).abs() < 1e-4 + 1e-4 * b.abs(),
4237 "{what}[{index}]: {a} against {b}"
4238 );
4239 }
4240 };
4241 close(
4242 &output.download().unwrap().tokens.data,
4243 &activated.tokens.data,
4244 "output",
4245 );
4246 close(
4247 &input_grad.download().unwrap().tokens.data,
4248 &expected_grad.tokens.data,
4249 "input grad",
4250 );
4251 resident.to_cpu().unwrap();
4252 close(
4253 &resident.gamma.grad.data,
4254 &host.gamma.grad.data,
4255 "gamma grad",
4256 );
4257 close(&resident.beta.grad.data, &host.beta.grad.data, "beta grad");
4258 }
4259
4260 #[cfg(feature = "cuda")]
4262 #[test]
4263 fn device_spatial_softmax_and_pooling_match_the_host() {
4264 let Some(device) = cuda_or_skip() else {
4265 return;
4266 };
4267 let input = noise(3, 17, 19, 5, 1);
4268 let (expected, cache) = spatial_softmax(&input);
4269 let grad = noise(3, 1, 1, 10, 2).tokens;
4270 let expected_grad = spatial_softmax_backward(&cache, &grad);
4271 let pooled = global_avg_pool(&input);
4272 let pool_grad = noise(3, 1, 1, 5, 3).tokens;
4273 let expected_pool_grad = global_avg_pool_backward(&pool_grad, 17, 19);
4274
4275 let context = device.context();
4276 let on_card = device.upload(&input).unwrap();
4277 let on_card_grad = device
4278 .upload(&ImageBatch::from_tokens(3, 1, 1, grad))
4279 .unwrap();
4280 let on_card_pool_grad = device
4281 .upload(&ImageBatch::from_tokens(3, 1, 1, pool_grad))
4282 .unwrap();
4283 let before = context.transfers();
4284 let (output, cache) = spatial_softmax_device(&on_card).unwrap();
4285 let back = spatial_softmax_backward_device(&cache, &on_card_grad).unwrap();
4286 let mean = global_avg_pool_device(&on_card).unwrap();
4287 let spread = global_avg_pool_backward_device(&on_card_pool_grad, 17, 19).unwrap();
4288 assert_eq!(
4289 context.transfers(),
4290 before,
4291 "nothing crosses inside the chain"
4292 );
4293
4294 let close = |a: &[f32], b: &[f32], what: &str| {
4295 assert_eq!(a.len(), b.len(), "{what}");
4296 for (index, (a, b)) in a.iter().zip(b).enumerate() {
4297 assert!(
4298 (a - b).abs() < 1e-5 + 1e-4 * b.abs(),
4299 "{what}[{index}]: {a} against {b}"
4300 );
4301 }
4302 };
4303 close(
4304 &output.download().unwrap().tokens.data,
4305 &expected.data,
4306 "keypoints",
4307 );
4308 close(
4309 &back.download().unwrap().tokens.data,
4310 &expected_grad.tokens.data,
4311 "keypoint grad",
4312 );
4313 close(&mean.download().unwrap().tokens.data, &pooled.data, "pool");
4314 close(
4315 &spread.download().unwrap().tokens.data,
4316 &expected_pool_grad.tokens.data,
4317 "pool grad",
4318 );
4319 }
4320
4321 #[cfg(feature = "cuda")]
4324 #[test]
4325 fn im2col_on_the_device_builds_the_same_columns_as_the_host() {
4326 let Some(device) = cuda_or_skip() else {
4327 return;
4328 };
4329 let mut rng = rand::rngs::StdRng::seed_from_u64(13);
4330 let conv = TrainableConv2d::new(5, 7, 3, 2, &mut rng).unwrap();
4331 let input = noise(2, 9, 11, 5, 6);
4332 let shape = conv.geometry(input.batch, input.height, input.width);
4333 let expected = conv.im2col(&input, &shape);
4334
4335 let context = device.context();
4336 let resident = context.upload(&input.tokens).unwrap();
4337 let columns = context.im2col(&resident, &shape).unwrap();
4338 let mut actual = Matrix::new(expected.rows, expected.cols);
4339 context.download(&columns, &mut actual).unwrap();
4340 assert_eq!(actual.data, expected.data);
4341
4342 let scattered = context.col2im(&columns, &shape).unwrap();
4345 let mut actual = Matrix::new(input.tokens.rows, input.tokens.cols);
4346 context.download(&scattered, &mut actual).unwrap();
4347 let expected = conv.col2im(&expected, &shape);
4348 for (index, (a, b)) in actual.data.iter().zip(&expected.tokens.data).enumerate() {
4349 assert!(
4350 (a - b).abs() < 1e-5,
4351 "col2im[{index}]: {a} on the device against {b} on the host"
4352 );
4353 }
4354 }
4355
4356 #[cfg(feature = "cuda")]
4360 #[test]
4361 fn rebuilding_the_columns_on_the_device_reaches_the_same_gradients() {
4362 let Some(device) = cuda_or_skip() else {
4363 return;
4364 };
4365 let mut rng = rand::rngs::StdRng::seed_from_u64(41);
4366 let mut keeping = TrainableConv2d::new(16, 24, 3, 1, &mut rng).unwrap();
4367 let mut rebuilding = keeping.clone();
4368 rebuilding.set_recompute_columns(true);
4369 keeping.to_cuda_on(&device).unwrap();
4370 rebuilding.to_cuda_on(&device).unwrap();
4371 let input = noise(2, 16, 16, 16, 9);
4372
4373 let (kept, kept_cache) = keeping.forward(&input).unwrap();
4374 let (rebuilt, rebuilt_cache) = rebuilding.forward(&input).unwrap();
4375 assert_eq!(kept.tokens.data, rebuilt.tokens.data);
4376 assert!(!rebuilt_cache.holds_columns());
4377
4378 let grad_output = noise(2, kept.height, kept.width, 24, 10);
4379 let kept_input = keeping.backward(&kept_cache, &grad_output).unwrap();
4380 let rebuilt_input = rebuilding.backward(&rebuilt_cache, &grad_output).unwrap();
4381 assert_eq!(kept_input.tokens.data, rebuilt_input.tokens.data);
4382
4383 keeping.to_cpu().unwrap();
4384 rebuilding.to_cpu().unwrap();
4385 assert_eq!(
4386 keeping.weight.weight.grad.data,
4387 rebuilding.weight.weight.grad.data
4388 );
4389 assert_eq!(keeping.bias.grad.data, rebuilding.bias.grad.data);
4390 }
4391
4392 fn ramp(channels: usize, height: usize, width: usize) -> FeatureMap {
4393 let data = (0..channels * height * width)
4394 .map(|value| value as f32)
4395 .collect();
4396 FeatureMap::from_vec(channels, height, width, data).unwrap()
4397 }
4398
4399 #[test]
4400 fn quantizing_a_dense_keeps_its_answer_and_drops_its_floats() {
4401 let weight = Matrix::from_vec(
4402 3,
4403 4,
4404 (0..12).map(|index| (index as f32 * 0.7).sin()).collect(),
4405 );
4406 let tokens = Matrix::from_vec(2, 4, (0..8).map(|index| index as f32 * 0.25).collect());
4407 let mut layer = Dense::new(weight, Some(vec![0.5, -0.5, 0.25])).unwrap();
4408 let expected = layer.forward(&tokens).unwrap();
4409
4410 layer.quantize();
4411 assert!(layer.is_quantized());
4412 assert!(layer.weight.data.is_empty());
4413 assert_eq!((layer.out_dim(), layer.in_dim()), (3, 4));
4414
4415 let actual = layer.forward(&tokens).unwrap();
4416 for (actual, expected) in actual.data.iter().zip(&expected.data) {
4417 assert!(
4418 (actual - expected).abs() < 0.02,
4419 "{actual} is not {expected}"
4420 );
4421 }
4422 }
4423
4424 #[test]
4425 fn a_one_by_one_convolution_is_a_per_pixel_linear_layer() {
4426 let weight = Matrix::from_vec(2, 2, vec![1.0, 1.0, 0.0, -1.0]);
4429 let conv = Conv2d::new(weight, Some(vec![0.5, 0.0]), 2, 1, 1, 0).unwrap();
4430 let input = ramp(2, 2, 2);
4431
4432 let output = conv.forward(&input).unwrap();
4433
4434 assert_eq!(output.channels, 2);
4435 assert_eq!((output.height, output.width), (2, 2));
4436 assert_eq!(output.plane(0), &[4.5, 6.5, 8.5, 10.5]);
4439 assert_eq!(output.plane(1), &[-4.0, -5.0, -6.0, -7.0]);
4440 }
4441
4442 #[test]
4443 fn a_padded_three_by_three_convolution_keeps_the_size_and_reads_zeros_outside() {
4444 let mut weight = vec![0.0; 9];
4446 weight[1] = 1.0;
4447 let conv = Conv2d::new(Matrix::from_vec(1, 9, weight), None, 1, 3, 1, 1).unwrap();
4448 let input = ramp(1, 3, 3);
4449
4450 let output = conv.forward(&input).unwrap();
4451
4452 assert_eq!((output.height, output.width), (3, 3));
4453 assert_eq!(
4454 output.data,
4455 vec![0.0, 0.0, 0.0, 0.0, 1.0, 2.0, 3.0, 4.0, 5.0]
4456 );
4457 }
4458
4459 #[test]
4460 fn a_strided_convolution_halves_the_resolution() {
4461 let conv = Conv2d::new(Matrix::from_vec(1, 4, vec![0.25; 4]), None, 1, 2, 2, 0).unwrap();
4462 let output = conv.forward(&ramp(1, 4, 4)).unwrap();
4463
4464 assert_eq!((output.height, output.width), (2, 2));
4465 assert_eq!(output.data, vec![2.5, 4.5, 10.5, 12.5]);
4467 }
4468
4469 #[test]
4470 fn a_convolution_refuses_shapes_it_cannot_read() {
4471 let conv = Conv2d::new(Matrix::from_vec(1, 4, vec![1.0; 4]), None, 1, 2, 1, 0).unwrap();
4472 assert!(conv.forward(&ramp(2, 4, 4)).is_err());
4473 assert!(conv.forward(&ramp(1, 1, 1)).is_err());
4474 assert!(Conv2d::new(Matrix::from_vec(1, 3, vec![1.0; 3]), None, 1, 2, 1, 0).is_err());
4475 assert!(
4476 Conv2d::new(
4477 Matrix::from_vec(1, 4, vec![1.0; 4]),
4478 Some(vec![0.0; 2]),
4479 1,
4480 2,
4481 1,
4482 0
4483 )
4484 .is_err()
4485 );
4486 }
4487
4488 #[test]
4489 fn group_norm_standardizes_each_group_over_its_own_pixels() {
4490 let mut map = ramp(4, 2, 2);
4491 let norm = GroupNorm::new(2, vec![1.0; 4], vec![0.0; 4], 1e-5).unwrap();
4492 norm.forward(&mut map).unwrap();
4493
4494 for group in 0..2 {
4495 let values = &map.data[group * 8..(group + 1) * 8];
4496 let mean = values.iter().sum::<f32>() / 8.0;
4497 let variance = values
4498 .iter()
4499 .map(|value| (value - mean).powi(2))
4500 .sum::<f32>()
4501 / 8.0;
4502 assert!(mean.abs() < 1e-5, "{mean}");
4503 assert!((variance - 1.0).abs() < 1e-3, "{variance}");
4504 }
4505
4506 let mut map = ramp(2, 1, 2);
4508 let norm = GroupNorm::new(1, vec![2.0, 0.5], vec![1.0, -1.0], 1e-5).unwrap();
4509 norm.forward(&mut map).unwrap();
4510 assert!(
4511 (map.data[0] - (-1.341_640_8 * 2.0 + 1.0)).abs() < 1e-4,
4512 "{:?}",
4513 map.data
4514 );
4515
4516 assert!(GroupNorm::new(3, vec![1.0; 4], vec![0.0; 4], 1e-5).is_err());
4517 }
4518
4519 #[test]
4520 fn upsampling_repeats_pixels_and_shuffling_round_trips() {
4521 let map = ramp(1, 2, 2);
4522 let large = upsample_nearest(&map, 2);
4523 assert_eq!((large.height, large.width), (4, 4));
4524 assert_eq!(&large.data[..4], &[0.0, 0.0, 1.0, 1.0]);
4525 assert_eq!(&large.data[4..8], &[0.0, 0.0, 1.0, 1.0]);
4526
4527 let packed = ramp(8, 2, 3);
4528 let shuffled = pixel_shuffle(&packed, 2).unwrap();
4529 assert_eq!(
4530 (shuffled.channels, shuffled.height, shuffled.width),
4531 (2, 4, 6)
4532 );
4533 assert_eq!(pixel_unshuffle(&shuffled, 2).unwrap(), packed);
4534
4535 assert!(pixel_shuffle(&packed, 3).is_err());
4536 assert!(pixel_unshuffle(&ramp(1, 3, 3), 2).is_err());
4537 }
4538
4539 #[test]
4540 fn a_map_survives_the_trip_through_token_shape() {
4541 let map = ramp(3, 2, 4);
4542 let tokens = map.to_tokens();
4543 assert_eq!((tokens.rows, tokens.cols), (8, 3));
4544 assert_eq!(tokens.row(1), &[1.0, 9.0, 17.0]);
4545 assert_eq!(FeatureMap::from_tokens(&tokens, 2, 4).unwrap(), map);
4546 assert!(FeatureMap::from_tokens(&tokens, 3, 4).is_err());
4547 }
4548 fn dot(a: &[f32], b: &[f32]) -> f32 {
4549 a.iter().zip(b).map(|(a, b)| a * b).sum()
4550 }
4551
4552 fn batch_norm_for_tests() -> TrainableBatchNorm2d {
4553 let mut norm = TrainableBatchNorm2d::yolo(6);
4554 norm.gamma.value.data = noise(1, 1, 6, 1, 4)
4555 .tokens
4556 .data
4557 .iter()
4558 .map(|v| v + 1.5)
4559 .collect();
4560 norm.beta.value.data = noise(1, 1, 6, 1, 5).tokens.data;
4561 norm.running.value.data = noise(1, 2, 6, 1, 9)
4562 .tokens
4563 .data
4564 .iter()
4565 .enumerate()
4566 .map(|(i, v)| if i < 6 { *v } else { v.abs() + 0.5 })
4567 .collect();
4568 norm
4569 }
4570
4571 #[test]
4572 fn batch_norm_matches_finite_differences_in_both_modes_and_tracks_running_statistics() {
4573 let input = noise(2, 3, 5, 6, 3);
4574 let weights = noise(2, 3, 5, 6, 6);
4575 for training in [true, false] {
4576 let mut norm = batch_norm_for_tests();
4577 let before = norm.running.value.data.clone();
4578 let (output, cache) = norm.forward(&input, training).unwrap();
4579 let grad = norm.backward(&cache, &weights);
4580 let through = |input: &ImageBatch| {
4581 batch_norm_for_tests()
4582 .forward(input, training)
4583 .unwrap()
4584 .0
4585 .tokens
4586 };
4587 let gap = pooled_gap(through, &input, &weights.tokens.data, &grad.tokens.data);
4588 assert!(gap < 1e-3, "training {training}: input {gap}");
4589 let normalized: Vec<f32> = output
4591 .tokens
4592 .data
4593 .iter()
4594 .enumerate()
4595 .map(|(i, y)| (y - norm.beta.value.data[i % 6]) / norm.gamma.value.data[i % 6])
4596 .collect();
4597 for c in 0..6 {
4598 let (mut dg, mut db) = (0.0, 0.0);
4599 for r in 0..30 {
4600 dg += weights.tokens.data[r * 6 + c] * normalized[r * 6 + c];
4601 db += weights.tokens.data[r * 6 + c];
4602 }
4603 assert!((norm.gamma.grad.data[c] - dg).abs() < 1e-4);
4604 assert!((norm.beta.grad.data[c] - db).abs() < 1e-4);
4605 }
4606 if training {
4607 let c = 2;
4608 let column: Vec<f32> = (0..30).map(|r| input.tokens.data[r * 6 + c]).collect();
4609 let mean = column.iter().sum::<f32>() / 30.0;
4610 let unbiased = column.iter().map(|v| (v - mean).powi(2)).sum::<f32>() / 29.0;
4611 let running = &norm.running.value.data;
4612 assert!((running[c] - (0.97 * before[c] + 0.03 * mean)).abs() < 1e-6);
4613 assert!((running[6 + c] - (0.97 * before[6 + c] + 0.03 * unbiased)).abs() < 1e-6);
4614 } else {
4615 assert_eq!(norm.running.value.data, before);
4616 }
4617 }
4618 }
4619
4620 #[test]
4621 fn a_fused_convolution_equals_the_convolution_then_the_evaluating_norm() {
4622 let mut rng = rand::rngs::StdRng::seed_from_u64(3);
4623 let conv = TrainableConv2d::new(4, 6, 3, 2, &mut rng)
4624 .unwrap()
4625 .without_bias();
4626 let mut norm = batch_norm_for_tests();
4627 let input = noise(2, 7, 9, 4, 1);
4628 let (expected, _) = norm
4629 .forward(&conv.forward(&input).unwrap().0, false)
4630 .unwrap();
4631 let fused = norm.fuse_into(&conv).forward(&input).unwrap().0;
4632 for (a, b) in fused.tokens.data.iter().zip(&expected.tokens.data) {
4633 assert!((a - b).abs() < 1e-5, "{a} against {b}");
4634 }
4635 }
4636
4637 #[test]
4638 fn upsample_concat_split_and_max_pool_backwards_are_their_adjoints() {
4639 let input = noise(2, 3, 4, 5, 1);
4640 let up = upsample_nearest_batch(&input, 2);
4641 assert_eq!((up.height, up.width), (6, 8));
4642 assert_eq!(up.pixel(1, 5, 7), input.pixel(1, 2, 3));
4643 let grad = noise(2, 6, 8, 5, 2);
4644 let back = upsample_nearest_backward(&grad, 2);
4645 let gap =
4646 dot(&up.tokens.data, &grad.tokens.data) - dot(&input.tokens.data, &back.tokens.data);
4647 assert!(gap.abs() < 1e-3, "upsample {gap}");
4648
4649 let other = noise(2, 3, 4, 2, 7);
4650 let joined = concat_channels(&[&input, &other]);
4651 assert_eq!(joined.pixel(1, 2, 3)[5..], *other.pixel(1, 2, 3));
4652 let parts = split_channels(&joined, &[5, 2]);
4653 assert_eq!(parts[0].tokens.data, input.tokens.data);
4654 assert_eq!(parts[1].tokens.data, other.tokens.data);
4655
4656 let mut distinct = noise(2, 6, 7, 3, 4);
4658 distinct
4659 .tokens
4660 .data
4661 .iter_mut()
4662 .enumerate()
4663 .for_each(|(i, v)| *v += i as f32 * 1e-3);
4664 let (pooled, cache) = max_pool_same(&distinct, 5);
4665 assert_eq!(
4666 pooled.pixel(0, 0, 0)[0],
4667 (0..3)
4668 .flat_map(|y| (0..3).map(move |x| (y, x)))
4669 .map(|(y, x)| distinct.pixel(0, y, x)[0])
4670 .fold(f32::MIN, f32::max)
4671 );
4672 let weights = noise(2, 6, 7, 3, 8);
4673 let back = max_pool_same_backward(&cache, &weights);
4674 let through = |input: &ImageBatch| max_pool_same(input, 5).0.tokens;
4675 let gap = pooled_gap(through, &distinct, &weights.tokens.data, &back.tokens.data);
4676 assert!(gap < 1e-3, "max pool {gap}");
4677 }
4678
4679 #[cfg(feature = "cuda")]
4684 #[test]
4685 fn a_device_batch_norm_over_offset_values_matches_the_host() {
4686 let Some(device) = cuda_or_skip() else {
4687 return;
4688 };
4689 let mut input = noise(4, 48, 48, 6, 3);
4690 input.tokens.data.iter_mut().for_each(|v| *v += 30.0);
4691 let mut host = batch_norm_for_tests();
4692 let mut resident = batch_norm_for_tests();
4693 resident.to_cuda_on(&device).unwrap();
4694 let (expected, _) = host.forward(&input, true).unwrap();
4695 let (output, _) = resident
4696 .forward_device(&device.upload(&input).unwrap(), true)
4697 .unwrap();
4698 let output = output.download().unwrap().tokens.data;
4699 let gap = output
4700 .iter()
4701 .zip(&expected.tokens.data)
4702 .map(|(a, b)| (a - b).abs())
4703 .fold(0.0f32, f32::max);
4704 assert!(gap < 5e-4, "normalized values {gap} apart");
4705 }
4706
4707 #[cfg(feature = "cuda")]
4709 #[test]
4710 fn device_detection_layers_match_the_host() {
4711 let Some(device) = cuda_or_skip() else {
4712 return;
4713 };
4714 let close = |a: &[f32], b: &[f32], what: &str| {
4715 assert_eq!(a.len(), b.len(), "{what}");
4716 for (index, (a, b)) in a.iter().zip(b).enumerate() {
4717 assert!(
4718 (a - b).abs() < 1e-4 + 1e-4 * b.abs(),
4719 "{what}[{index}]: {a} against {b}"
4720 );
4721 }
4722 };
4723 let input = noise(3, 9, 11, 6, 1);
4724 let weights = noise(3, 9, 11, 6, 2);
4725 let on_card = device.upload(&input).unwrap();
4726 let on_card_weights = device.upload(&weights).unwrap();
4727 for training in [true, false] {
4728 let mut host = batch_norm_for_tests();
4729 let mut resident = batch_norm_for_tests();
4730 resident.to_cuda_on(&device).unwrap();
4731 let (expected, cache) = host.forward(&input, training).unwrap();
4732 let expected_grad = host.backward(&cache, &weights);
4733 let (output, cache) = resident.forward_device(&on_card, training).unwrap();
4734 let grad = resident.backward_device(&cache, &on_card_weights).unwrap();
4735 close(
4736 &output.download().unwrap().tokens.data,
4737 &expected.tokens.data,
4738 "norm",
4739 );
4740 close(
4741 &grad.download().unwrap().tokens.data,
4742 &expected_grad.tokens.data,
4743 "norm grad",
4744 );
4745 resident.to_cpu().unwrap();
4746 close(
4747 &resident.gamma.grad.data,
4748 &host.gamma.grad.data,
4749 "gamma grad",
4750 );
4751 close(&resident.beta.grad.data, &host.beta.grad.data, "beta grad");
4752 close(
4753 &resident.running.value.data,
4754 &host.running.value.data,
4755 "running",
4756 );
4757 }
4758
4759 let up = upsample_nearest_device(&on_card, 2).unwrap();
4760 close(
4761 &up.download().unwrap().tokens.data,
4762 &upsample_nearest_batch(&input, 2).tokens.data,
4763 "upsample",
4764 );
4765 let back = upsample_nearest_backward_device(&up, 2).unwrap();
4766 close(
4767 &back.download().unwrap().tokens.data,
4768 &upsample_nearest_backward(&upsample_nearest_batch(&input, 2), 2)
4769 .tokens
4770 .data,
4771 "upsample grad",
4772 );
4773
4774 let joined = concat_channels_device(&[&on_card, &on_card_weights]).unwrap();
4775 close(
4776 &joined.download().unwrap().tokens.data,
4777 &concat_channels(&[&input, &weights]).tokens.data,
4778 "concat",
4779 );
4780 let parts = split_channels_device(&joined, &[4, 8]).unwrap();
4781 for (part, expected) in parts.iter().zip(split_channels(
4782 &concat_channels(&[&input, &weights]),
4783 &[4, 8],
4784 )) {
4785 close(
4786 &part.download().unwrap().tokens.data,
4787 &expected.tokens.data,
4788 "split",
4789 );
4790 }
4791 let sum = add_device(&on_card, &on_card_weights).unwrap();
4792 let mut expected = input.clone();
4793 expected.add(&weights);
4794 close(
4795 &sum.download().unwrap().tokens.data,
4796 &expected.tokens.data,
4797 "add",
4798 );
4799
4800 let (pooled, cache) = max_pool_same(&input, 5);
4801 let (on_card_pooled, device_cache) = max_pool_same_device(&on_card, 5).unwrap();
4802 close(
4803 &on_card_pooled.download().unwrap().tokens.data,
4804 &pooled.tokens.data,
4805 "max pool",
4806 );
4807 let back = max_pool_same_backward_device(&device_cache, &on_card_weights).unwrap();
4808 close(
4809 &back.download().unwrap().tokens.data,
4810 &max_pool_same_backward(&cache, &weights).tokens.data,
4811 "max pool grad",
4812 );
4813 }
4814 #[test]
4815 fn depthwise_convolution_matches_finite_differences() {
4816 let mut rng = rand::rngs::StdRng::seed_from_u64(5);
4817 for stride in [1, 2] {
4818 let mut conv = DepthwiseConv2d::new(4, 3, stride, &mut rng);
4819 let input = noise(2, 7, 6, 4, 1);
4820 let (output, cache) = conv.forward(&input).unwrap();
4821 assert_eq!((output.height, output.width), conv.output_size(7, 6));
4822 let w = &conv.weight.value.data[9..18];
4824 let (y, x) = (1, 2 / stride);
4825 let direct: f32 = (0..9)
4826 .map(|t| input.pixel(1, y * stride + t / 3 - 1, x * stride + t % 3 - 1)[1] * w[t])
4827 .sum();
4828 assert!((output.pixel(1, y, x)[1] - direct).abs() < 1e-5);
4829 let weights = noise(2, output.height, output.width, 4, 6);
4830 let grad = conv.backward(&cache, &weights);
4831 let through = |input: &ImageBatch| conv.forward(input).unwrap().0.tokens;
4832 let gap = pooled_gap(through, &input, &weights.tokens.data, &grad.tokens.data);
4833 assert!(gap < 1e-3, "stride {stride}: input {gap}");
4834 let as_batch =
4835 |data: Vec<f32>| ImageBatch::from_tokens(1, 1, 36, Matrix::from_vec(36, 1, data));
4836 let through = |params: &ImageBatch| {
4837 let mut moved = conv.clone();
4838 moved.weight.value.data.copy_from_slice(¶ms.tokens.data);
4839 moved.forward(&input).unwrap().0.tokens
4840 };
4841 let gap = pooled_gap(
4842 through,
4843 &as_batch(conv.weight.value.data.clone()),
4844 &weights.tokens.data,
4845 &conv.weight.grad.data,
4846 );
4847 assert!(gap < 1e-3, "stride {stride}: weight {gap}");
4848 }
4849 }
4850
4851 #[cfg(feature = "cuda")]
4852 #[test]
4853 fn device_depthwise_convolution_matches_the_host() {
4854 let Some(device) = cuda_or_skip() else {
4855 return;
4856 };
4857 let mut rng = rand::rngs::StdRng::seed_from_u64(5);
4858 for stride in [1, 2] {
4859 let mut host = DepthwiseConv2d::new(6, 3, stride, &mut rng);
4860 let mut resident = host.clone();
4861 resident.to_cuda_on(&device).unwrap();
4862 let input = noise(3, 13, 10, 6, 1);
4863 let (expected, cache) = host.forward(&input).unwrap();
4864 let weights = noise(3, expected.height, expected.width, 6, 2);
4865 let expected_grad = host.backward(&cache, &weights);
4866 let on_card = device.upload(&input).unwrap();
4867 let output = resident.forward_device(&on_card).unwrap();
4868 let grad = resident
4869 .backward_device(&on_card, &device.upload(&weights).unwrap())
4870 .unwrap();
4871 let close = |a: &[f32], b: &[f32], what: &str| {
4872 assert_eq!(a.len(), b.len(), "{what}");
4873 for (index, (a, b)) in a.iter().zip(b).enumerate() {
4874 assert!(
4875 (a - b).abs() < 1e-4 + 1e-4 * b.abs(),
4876 "{what}[{index}]: {a} against {b}"
4877 );
4878 }
4879 };
4880 close(
4881 &output.download().unwrap().tokens.data,
4882 &expected.tokens.data,
4883 "output",
4884 );
4885 close(
4886 &grad.download().unwrap().tokens.data,
4887 &expected_grad.tokens.data,
4888 "input grad",
4889 );
4890 resident.to_cpu().unwrap();
4891 close(
4892 &resident.weight.grad.data,
4893 &host.weight.grad.data,
4894 "weight grad",
4895 );
4896 }
4897 }
4898
4899 #[test]
4900 fn pixel_attention_matches_ultralytics() {
4901 let fixture = std::path::Path::new("/home/vasylt/Code/games/cs2/fixtures/k13a_attention.json");
4902 let Ok(text) = std::fs::read_to_string(fixture) else {
4903 eprintln!("skipped: no {}", fixture.display());
4904 return;
4905 };
4906 let json: serde_json::Value = serde_json::from_str(&text).unwrap();
4907 let size = |key: &str| json[key].as_u64().unwrap() as usize;
4908 let map = |key: &str| {
4909 let data: Vec<f32> = serde_json::from_value(json[key].clone()).unwrap();
4910 let rows = size("batch") * size("height") * size("width");
4911 let tokens = Matrix::from_vec(rows, data.len() / rows, data);
4912 ImageBatch::from_tokens(size("batch"), size("height"), size("width"), tokens)
4913 };
4914 let shape = PixelAttentionShape::yolo(size("dim"), size("heads"));
4915 let (attended, values, cache) = pixel_attention(&map("qkv"), shape).unwrap();
4916 let grad = pixel_attention_backward(&cache, &map("grad_attended"), &map("grad_values"));
4917 for (got, key) in [
4918 (attended, "attended"),
4919 (values, "values"),
4920 (grad, "grad_qkv"),
4921 ] {
4922 let gap = got
4923 .tokens
4924 .data
4925 .iter()
4926 .zip(&map(key).tokens.data)
4927 .fold(0.0f32, |m, (a, b)| m.max((a - b).abs()));
4928 assert!(gap < 1e-5, "{key}: {gap}");
4929 }
4930 }
4931
4932 #[cfg(feature = "cuda")]
4933 #[test]
4934 fn device_pixel_attention_matches_the_host() {
4935 let Some(device) = cuda_or_skip() else {
4936 return;
4937 };
4938 let shape = PixelAttentionShape::yolo(16, 2);
4939 let qkv = noise(3, 5, 4, 32, 1);
4940 let (attended, values, cache) = pixel_attention(&qkv, shape).unwrap();
4941 let (grad_attended, grad_values) = (noise(3, 5, 4, 16, 2), noise(3, 5, 4, 16, 3));
4942 let grad = pixel_attention_backward(&cache, &grad_attended, &grad_values);
4943 let (on_attended, on_values, on_cache) =
4944 pixel_attention_device(&device.upload(&qkv).unwrap(), shape).unwrap();
4945 let on_grad = pixel_attention_backward_device(
4946 &on_cache,
4947 &device.upload(&grad_attended).unwrap(),
4948 &device.upload(&grad_values).unwrap(),
4949 )
4950 .unwrap();
4951 for (got, expected, what) in [
4952 (on_attended, attended, "attended"),
4953 (on_values, values, "values"),
4954 (on_grad, grad, "qkv grad"),
4955 ] {
4956 let got = got.download().unwrap();
4957 for (index, (a, b)) in got
4958 .tokens
4959 .data
4960 .iter()
4961 .zip(&expected.tokens.data)
4962 .enumerate()
4963 {
4964 assert!(
4965 (a - b).abs() < 1e-4 + 1e-4 * b.abs(),
4966 "{what}[{index}]: {a} against {b}"
4967 );
4968 }
4969 }
4970 }
4971}