bunsen 0.42.0

bunsen is a batteries included common library for burn
Documentation
//! # Patch Merging
use burn::{
    config::Config,
    module::Module,
    nn::{
        LayerNorm,
        LayerNormConfig,
        Linear,
        LinearConfig,
    },
    prelude::Tensor,
    tensor::Device,
};

use crate::{
    burner::module::ModuleInit,
    contracts::{
        assert_shape_contract_periodically,
        unpack_shape_contract,
    },
    errors::{
        BunsenResult,
        ConstraintError,
        Rule,
    },
    kits::images::swin::ops::collate_patches,
};

/// Metadata for `PatchMerging`.
pub trait PatchMergingMeta {
    /// Input feature dimension size.
    fn d_input(&self) -> usize;

    /// Output feature dimension size.
    fn d_output(&self) -> usize {
        2 * self.d_input()
    }

    /// Input resolution (height, width).
    fn input_resolution(&self) -> [usize; 2];

    /// Input height.
    fn input_height(&self) -> usize {
        self.input_resolution()[0]
    }

    /// Input width.
    fn input_width(&self) -> usize {
        self.input_resolution()[1]
    }

    /// Output resolution (height, width).
    fn output_resolution(&self) -> [usize; 2] {
        [self.output_height(), self.output_width()]
    }

    /// Output height.
    fn output_height(&self) -> usize {
        self.input_height() / 2
    }

    /// Output width.
    fn output_width(&self) -> usize {
        self.input_width() / 2
    }
}

/// Configuration for [`PatchMerging`].
///
/// Specifies the input resolution and feature dimension for a patch-merging
/// downsample step. Build the module with `.init(device)`, then `forward` a
/// `[batch, height * width, channels]` tensor to halve the spatial resolution
/// and double the channels.
#[derive(Config, Debug)]
pub struct PatchMergingConfig {
    /// Input resolution (height, width).
    input_resolution: [usize; 2],

    /// Input feature dimension size.
    d_input: usize,
}

impl PatchMergingMeta for PatchMergingConfig {
    fn d_input(&self) -> usize {
        self.d_input
    }

    fn input_resolution(&self) -> [usize; 2] {
        self.input_resolution
    }
}

impl ModuleInit<PatchMerging> for PatchMergingConfig {
    fn try_init(
        &self,
        device: &Device,
    ) -> BunsenResult<PatchMerging> {
        for (axis, value) in self.input_resolution.into_iter().enumerate() {
            if !value.is_multiple_of(2) {
                return Err(ConstraintError::new(
                    "PatchMergingConfig",
                    format!("input_resolution[{axis}]"),
                    Rule::NotMultiple {
                        value,
                        of: "2".into(),
                    },
                )
                .into());
            }
        }

        Ok(PatchMerging {
            input_resolution: self.input_resolution,
            reduction: LinearConfig::new(2 * self.d_output(), self.d_output())
                .with_bias(false)
                .init(device),
            norm: LayerNormConfig::new(self.d_output()).init(device),
        })
    }
}

/// SWIN-Transformer v2 `PatchMerging` module.
///
/// This module accepts `[batch, height * width, channels]` inputs, and then:
/// - Collates interleaved patches of size `[height/2, width/2]` into `[batch,
///   height/2 * width/2, 4 * channels]`.
/// - Applies a linear layer to reduce the feature dimension to ``2 *
///   channels``.
/// - Applies layer normalization.
/// - Yields `[batch, height/2 * width/2, 2 * channels]` output.
///
/// See: <https://github.com/microsoft/Swin-Transformer/blob/main/models/swin_transformer_v2.py>
///
/// Built by [`PatchMergingConfig`].
#[derive(Module, Debug)]
pub struct PatchMerging {
    /// Input resolution (height, width).
    input_resolution: [usize; 2],

    /// Linear layer for reducing the feature dimension.
    reduction: Linear,

    /// Layer norm layer.
    norm: LayerNorm,
}

impl PatchMergingMeta for PatchMerging {
    fn d_input(&self) -> usize {
        self.reduction.weight.dims()[0] / 4
    }

    fn input_resolution(&self) -> [usize; 2] {
        self.input_resolution
    }
}

impl PatchMerging {
    /// Forward pass of the `PatchMerging` module.
    ///
    /// # Arguments
    ///
    /// - `x` - `[batch, height * width, channels]` input tensor.
    ///
    /// # Returns
    ///
    /// A `[batch, height/2 * width/2, 2 * channels]` tensor.
    ///
    /// # Panics
    ///
    /// On shape contract failure.
    pub fn forward(
        &self,
        x: Tensor<3>,
    ) -> Tensor<3> {
        let [b, h, w] = unpack_shape_contract!(
            ["batch", "flat" = "height" * "width", "d_in"],
            &x.dims(),
            &["batch", "height", "width"],
            &[
                ("height", self.input_height()),
                ("width", self.input_width()),
                ("d_in", self.d_input()),
            ]
        );

        let x = collate_patches(x, h, w);

        let x = self.reduction.forward(x);

        let x = self.norm.forward(x);
        assert_shape_contract_periodically!(
            ["batch", "flat" = "half_height" * "half_width", "d_out"],
            &x.dims(),
            &[
                ("batch", b),
                ("half_height", self.output_height()),
                ("half_width", self.output_width()),
                ("d_out", self.d_output()),
            ]
        );

        x
    }
}

#[cfg(test)]
mod tests {
    use burn::tensor::{
        Distribution,
        Tolerance,
    };
    use serial_test::serial;

    use super::*;
    use crate::{
        blocks::images::patching::patch_embed::{
            PatchEmbed,
            PatchEmbedConfig,
            PatchEmbedMeta,
        },
        errors::WithOkOrPanic,
        support::testing::{
            DeviceMemoryGuard,
            cpu_device,
            performance_device,
        },
    };

    #[test]
    #[serial]
    fn test_patch_merging_meta() {
        let config = PatchMergingConfig {
            input_resolution: [12, 8],
            d_input: 3,
        };

        assert_eq!(config.input_resolution(), [12, 8]);
        assert_eq!(config.d_input(), 3);
        assert_eq!(config.d_output(), 6);
        assert_eq!(config.output_resolution(), [6, 4]);
        assert_eq!(config.output_height(), 6);
        assert_eq!(config.output_width(), 4);

        let device = performance_device();
        let _memory = DeviceMemoryGuard::new(&device);
        let patch_merging: PatchMerging = config.try_init(&device).ok_or_panic();

        assert_eq!(patch_merging.input_resolution(), [12, 8]);
        assert_eq!(patch_merging.d_input(), 3);
        assert_eq!(patch_merging.d_output(), 6);
        assert_eq!(patch_merging.output_resolution(), [6, 4]);
        assert_eq!(patch_merging.output_height(), 6);
        assert_eq!(patch_merging.output_width(), 4);
    }

    #[should_panic(expected = "PatchMergingConfig.input_resolution[0]: 13 is not a multiple of 2")]
    #[test]
    #[serial]
    fn test_patch_merging_invalid_resolution() {
        let config = PatchMergingConfig {
            input_resolution: [13, 8], // Invalid height
            d_input: 3,
        };
        let device = performance_device();
        let _memory = DeviceMemoryGuard::new(&device);
        let _d: PatchMerging = config.try_init(&device).ok_or_panic();
    }

    #[test]
    #[serial]
    fn test_patch_merging() {
        let device = performance_device();
        let _memory = DeviceMemoryGuard::new(&device);

        let b = 2;
        let h = 12;
        let w = 8;
        let c = 3;

        let config = PatchMergingConfig {
            input_resolution: [h, w],
            d_input: c,
        };
        let patch_merging: PatchMerging = config.try_init(&device).ok_or_panic();

        let distribution = Distribution::Normal(0., 1.);
        let x = Tensor::random([b, h * w, c], distribution, &device);

        let y = patch_merging.forward(x.clone());
        assert_eq!(&y.dims(), &[b, h / 2 * w / 2, 2 * c]);
    }

    #[test]
    fn test_patch_embed_meta() {
        let config = PatchEmbedConfig::new([12, 8], 4, 3, 6).with_enable_patch_norm(false);

        assert_eq!(config.input_resolution(), [12, 8]);
        assert_eq!(config.patch_size(), 4);
        assert_eq!(config.d_input(), 3);
        assert_eq!(config.d_output(), 6);
        assert!(!config.enable_patch_norm());
        assert_eq!(config.patches_resolution(), [3, 2]);
        assert_eq!(config.patches_height(), 3);
        assert_eq!(config.patches_width(), 2);

        let device = cpu_device();
        let patch_embed: PatchEmbed = config.try_init(&device).ok_or_panic();

        assert_eq!(patch_embed.input_resolution(), [12, 8]);
        assert_eq!(patch_embed.patch_size(), 4);
        assert_eq!(patch_embed.d_input(), 3);
        assert_eq!(patch_embed.d_output(), 6);
        assert!(!patch_embed.enable_patch_norm());
        assert_eq!(patch_embed.patches_resolution(), [3, 2]);
        assert_eq!(patch_embed.patches_height(), 3);
        assert_eq!(patch_embed.patches_width(), 2);
    }

    #[test]
    #[serial]
    fn test_patch_embed() {
        let device = performance_device();
        let _memory = DeviceMemoryGuard::new(&device);

        let b = 2;
        let h = 12;
        let w = 8;

        let patch_size = 4;
        let d_input = 3;
        let d_output = d_input * 2;

        let distribution = Distribution::Normal(0., 1.);
        let x = Tensor::random([b, d_input, h, w], distribution, &device);

        // W/O Norm.
        {
            let config = PatchEmbedConfig::new([h, w], patch_size, d_input, d_output)
                .with_enable_patch_norm(false);
            let patch_embed = config.try_init(&device).ok_or_panic();

            let y = patch_embed.forward(x.clone());
            assert_eq!(&y.dims(), &[b, (h / 4) * (w / 4), d_output]);

            let z = patch_embed.projection.forward(x.clone());
            let z: Tensor<3> = z.flatten(2, 3);
            let z = z.swap_dims(1, 2);

            y.into_data_as::<f32>()
                .assert_approx_eq::<f32>(&z.into_data_as::<f32>(), Tolerance::default());
        }

        // With Norm.
        {
            let config = PatchEmbedConfig::new([h, w], patch_size, d_input, d_output);
            let patch_embed = config.try_init(&device).ok_or_panic();

            let y = patch_embed.forward(x.clone());
            assert_eq!(&y.dims(), &[b, (h / 4) * (w / 4), d_output]);

            let z = patch_embed.projection.forward(x.clone());
            let z: Tensor<3> = z.flatten(2, 3);
            let z = z.swap_dims(1, 2);
            let z = patch_embed.norm.as_ref().unwrap().forward(z);

            y.into_data_as::<f32>()
                .assert_approx_eq::<f32>(&z.into_data_as::<f32>(), Tolerance::default());
        }
    }
}