use burn::{
config::Config,
module::Module,
nn::{
LayerNorm,
LayerNormConfig,
Linear,
LinearConfig,
},
prelude::Tensor,
tensor::Device,
};
use crate::{
burner::module::ModuleInit,
contracts::{
assert_shape_contract_periodically,
unpack_shape_contract,
},
errors::{
BunsenResult,
ConstraintError,
Rule,
},
kits::images::swin::ops::collate_patches,
};
pub trait PatchMergingMeta {
fn d_input(&self) -> usize;
fn d_output(&self) -> usize {
2 * self.d_input()
}
fn input_resolution(&self) -> [usize; 2];
fn input_height(&self) -> usize {
self.input_resolution()[0]
}
fn input_width(&self) -> usize {
self.input_resolution()[1]
}
fn output_resolution(&self) -> [usize; 2] {
[self.output_height(), self.output_width()]
}
fn output_height(&self) -> usize {
self.input_height() / 2
}
fn output_width(&self) -> usize {
self.input_width() / 2
}
}
#[derive(Config, Debug)]
pub struct PatchMergingConfig {
input_resolution: [usize; 2],
d_input: usize,
}
impl PatchMergingMeta for PatchMergingConfig {
fn d_input(&self) -> usize {
self.d_input
}
fn input_resolution(&self) -> [usize; 2] {
self.input_resolution
}
}
impl ModuleInit<PatchMerging> for PatchMergingConfig {
fn try_init(
&self,
device: &Device,
) -> BunsenResult<PatchMerging> {
for (axis, value) in self.input_resolution.into_iter().enumerate() {
if !value.is_multiple_of(2) {
return Err(ConstraintError::new(
"PatchMergingConfig",
format!("input_resolution[{axis}]"),
Rule::NotMultiple {
value,
of: "2".into(),
},
)
.into());
}
}
Ok(PatchMerging {
input_resolution: self.input_resolution,
reduction: LinearConfig::new(2 * self.d_output(), self.d_output())
.with_bias(false)
.init(device),
norm: LayerNormConfig::new(self.d_output()).init(device),
})
}
}
#[derive(Module, Debug)]
pub struct PatchMerging {
input_resolution: [usize; 2],
reduction: Linear,
norm: LayerNorm,
}
impl PatchMergingMeta for PatchMerging {
fn d_input(&self) -> usize {
self.reduction.weight.dims()[0] / 4
}
fn input_resolution(&self) -> [usize; 2] {
self.input_resolution
}
}
impl PatchMerging {
pub fn forward(
&self,
x: Tensor<3>,
) -> Tensor<3> {
let [b, h, w] = unpack_shape_contract!(
["batch", "flat" = "height" * "width", "d_in"],
&x.dims(),
&["batch", "height", "width"],
&[
("height", self.input_height()),
("width", self.input_width()),
("d_in", self.d_input()),
]
);
let x = collate_patches(x, h, w);
let x = self.reduction.forward(x);
let x = self.norm.forward(x);
assert_shape_contract_periodically!(
["batch", "flat" = "half_height" * "half_width", "d_out"],
&x.dims(),
&[
("batch", b),
("half_height", self.output_height()),
("half_width", self.output_width()),
("d_out", self.d_output()),
]
);
x
}
}
#[cfg(test)]
mod tests {
use burn::tensor::{
Distribution,
Tolerance,
};
use serial_test::serial;
use super::*;
use crate::{
blocks::images::patching::patch_embed::{
PatchEmbed,
PatchEmbedConfig,
PatchEmbedMeta,
},
errors::WithOkOrPanic,
support::testing::{
DeviceMemoryGuard,
cpu_device,
performance_device,
},
};
#[test]
#[serial]
fn test_patch_merging_meta() {
let config = PatchMergingConfig {
input_resolution: [12, 8],
d_input: 3,
};
assert_eq!(config.input_resolution(), [12, 8]);
assert_eq!(config.d_input(), 3);
assert_eq!(config.d_output(), 6);
assert_eq!(config.output_resolution(), [6, 4]);
assert_eq!(config.output_height(), 6);
assert_eq!(config.output_width(), 4);
let device = performance_device();
let _memory = DeviceMemoryGuard::new(&device);
let patch_merging: PatchMerging = config.try_init(&device).ok_or_panic();
assert_eq!(patch_merging.input_resolution(), [12, 8]);
assert_eq!(patch_merging.d_input(), 3);
assert_eq!(patch_merging.d_output(), 6);
assert_eq!(patch_merging.output_resolution(), [6, 4]);
assert_eq!(patch_merging.output_height(), 6);
assert_eq!(patch_merging.output_width(), 4);
}
#[should_panic(expected = "PatchMergingConfig.input_resolution[0]: 13 is not a multiple of 2")]
#[test]
#[serial]
fn test_patch_merging_invalid_resolution() {
let config = PatchMergingConfig {
input_resolution: [13, 8], d_input: 3,
};
let device = performance_device();
let _memory = DeviceMemoryGuard::new(&device);
let _d: PatchMerging = config.try_init(&device).ok_or_panic();
}
#[test]
#[serial]
fn test_patch_merging() {
let device = performance_device();
let _memory = DeviceMemoryGuard::new(&device);
let b = 2;
let h = 12;
let w = 8;
let c = 3;
let config = PatchMergingConfig {
input_resolution: [h, w],
d_input: c,
};
let patch_merging: PatchMerging = config.try_init(&device).ok_or_panic();
let distribution = Distribution::Normal(0., 1.);
let x = Tensor::random([b, h * w, c], distribution, &device);
let y = patch_merging.forward(x.clone());
assert_eq!(&y.dims(), &[b, h / 2 * w / 2, 2 * c]);
}
#[test]
fn test_patch_embed_meta() {
let config = PatchEmbedConfig::new([12, 8], 4, 3, 6).with_enable_patch_norm(false);
assert_eq!(config.input_resolution(), [12, 8]);
assert_eq!(config.patch_size(), 4);
assert_eq!(config.d_input(), 3);
assert_eq!(config.d_output(), 6);
assert!(!config.enable_patch_norm());
assert_eq!(config.patches_resolution(), [3, 2]);
assert_eq!(config.patches_height(), 3);
assert_eq!(config.patches_width(), 2);
let device = cpu_device();
let patch_embed: PatchEmbed = config.try_init(&device).ok_or_panic();
assert_eq!(patch_embed.input_resolution(), [12, 8]);
assert_eq!(patch_embed.patch_size(), 4);
assert_eq!(patch_embed.d_input(), 3);
assert_eq!(patch_embed.d_output(), 6);
assert!(!patch_embed.enable_patch_norm());
assert_eq!(patch_embed.patches_resolution(), [3, 2]);
assert_eq!(patch_embed.patches_height(), 3);
assert_eq!(patch_embed.patches_width(), 2);
}
#[test]
#[serial]
fn test_patch_embed() {
let device = performance_device();
let _memory = DeviceMemoryGuard::new(&device);
let b = 2;
let h = 12;
let w = 8;
let patch_size = 4;
let d_input = 3;
let d_output = d_input * 2;
let distribution = Distribution::Normal(0., 1.);
let x = Tensor::random([b, d_input, h, w], distribution, &device);
{
let config = PatchEmbedConfig::new([h, w], patch_size, d_input, d_output)
.with_enable_patch_norm(false);
let patch_embed = config.try_init(&device).ok_or_panic();
let y = patch_embed.forward(x.clone());
assert_eq!(&y.dims(), &[b, (h / 4) * (w / 4), d_output]);
let z = patch_embed.projection.forward(x.clone());
let z: Tensor<3> = z.flatten(2, 3);
let z = z.swap_dims(1, 2);
y.into_data_as::<f32>()
.assert_approx_eq::<f32>(&z.into_data_as::<f32>(), Tolerance::default());
}
{
let config = PatchEmbedConfig::new([h, w], patch_size, d_input, d_output);
let patch_embed = config.try_init(&device).ok_or_panic();
let y = patch_embed.forward(x.clone());
assert_eq!(&y.dims(), &[b, (h / 4) * (w / 4), d_output]);
let z = patch_embed.projection.forward(x.clone());
let z: Tensor<3> = z.flatten(2, 3);
let z = z.swap_dims(1, 2);
let z = patch_embed.norm.as_ref().unwrap().forward(z);
y.into_data_as::<f32>()
.assert_approx_eq::<f32>(&z.into_data_as::<f32>(), Tolerance::default());
}
}
}