vit_tch 0.1.0

Vision Transformer models and building blocks for Rust using tch.
Documentation
use tch::{Kind, Tensor, nn};

use crate::{
    config::MaxViTConfig,
    conv_layers::MbConv,
    layers::{Attention, FeedForward},
    tensor::assert_image_patchable,
};

#[derive(Debug)]
struct MaxViTAttentionBlock {
    attention: Attention,
    feed_forward: FeedForward,
    window_size: i64,
    grid: bool,
}

impl MaxViTAttentionBlock {
    fn new(
        vs: &nn::Path,
        dim: i64,
        dim_head: i64,
        window_size: i64,
        dropout: f64,
        grid: bool,
    ) -> Self {
        assert_eq!(dim % dim_head, 0, "dim must be divisible by dim head");
        let heads = dim / dim_head;
        let attention = Attention::new(&(vs / "attention"), dim, heads, dim_head, dropout);
        let feed_forward = FeedForward::new(&(vs / "feed_forward"), dim, dim * 4, dropout);

        Self {
            attention,
            feed_forward,
            window_size,
            grid,
        }
    }

    fn block_tokens(&self, xs: &Tensor) -> (Tensor, i64, i64, i64, i64) {
        let size = xs.size();
        let batch = size[0];
        let dim = size[1];
        let height = size[2];
        let width = size[3];
        assert_image_patchable(height, width, self.window_size, self.window_size);
        let grid_h = height / self.window_size;
        let grid_w = width / self.window_size;
        let tokens = if self.grid {
            xs.view([
                batch,
                dim,
                self.window_size,
                grid_h,
                self.window_size,
                grid_w,
            ])
            .permute([0, 3, 5, 2, 4, 1])
            .contiguous()
        } else {
            xs.view([
                batch,
                dim,
                grid_h,
                self.window_size,
                grid_w,
                self.window_size,
            ])
            .permute([0, 2, 4, 3, 5, 1])
            .contiguous()
        };

        (
            tokens.view([
                batch * grid_h * grid_w,
                self.window_size * self.window_size,
                dim,
            ]),
            batch,
            height,
            width,
            dim,
        )
    }

    fn tokens_to_blocks(
        &self,
        tokens: &Tensor,
        batch: i64,
        height: i64,
        width: i64,
        dim: i64,
    ) -> Tensor {
        let grid_h = height / self.window_size;
        let grid_w = width / self.window_size;
        let tokens = tokens.view([
            batch,
            grid_h,
            grid_w,
            self.window_size,
            self.window_size,
            dim,
        ]);

        if self.grid {
            tokens
                .permute([0, 5, 3, 1, 4, 2])
                .contiguous()
                .view([batch, dim, height, width])
        } else {
            tokens
                .permute([0, 5, 1, 3, 2, 4])
                .contiguous()
                .view([batch, dim, height, width])
        }
    }
}

impl nn::ModuleT for MaxViTAttentionBlock {
    fn forward_t(&self, xs: &Tensor, train: bool) -> Tensor {
        let (tokens, batch, height, width, dim) = self.block_tokens(xs);
        let tokens = &tokens + self.attention.forward_t(&tokens, train);
        let ff_out = self.feed_forward.forward_t(&tokens, train);
        let tokens = tokens + ff_out;

        self.tokens_to_blocks(&tokens, batch, height, width, dim)
    }
}

#[derive(Debug)]
struct MaxViTLayer {
    mbconv: MbConv,
    block_attention: MaxViTAttentionBlock,
    grid_attention: MaxViTAttentionBlock,
}

impl MaxViTLayer {
    fn new(
        vs: &nn::Path,
        dim_in: i64,
        dim_out: i64,
        downsample: bool,
        config: &MaxViTConfig,
    ) -> Self {
        let stride = if downsample { 2 } else { 1 };
        let mbconv = MbConv::new(
            &(vs / "mbconv"),
            dim_in,
            dim_out,
            stride,
            config.mbconv_expansion_rate,
            config.mbconv_shrinkage_rate,
            config.dropout,
        );
        let block_attention = MaxViTAttentionBlock::new(
            &(vs / "block_attention"),
            dim_out,
            config.dim_head,
            config.window_size,
            config.dropout,
            false,
        );
        let grid_attention = MaxViTAttentionBlock::new(
            &(vs / "grid_attention"),
            dim_out,
            config.dim_head,
            config.window_size,
            config.dropout,
            true,
        );

        Self {
            mbconv,
            block_attention,
            grid_attention,
        }
    }
}

impl nn::ModuleT for MaxViTLayer {
    fn forward_t(&self, xs: &Tensor, train: bool) -> Tensor {
        let xs = self.mbconv.forward_t(xs, train);
        let xs = self.block_attention.forward_t(&xs, train);

        self.grid_attention.forward_t(&xs, train)
    }
}

#[derive(Debug)]
pub struct MaxViT {
    conv_stem1: nn::Conv2D,
    conv_stem2: nn::Conv2D,
    layers: Vec<MaxViTLayer>,
    head_norm: nn::LayerNorm,
    head: nn::Linear,
}

impl MaxViT {
    pub fn new(vs: &nn::Path, config: MaxViTConfig) -> Self {
        assert!(!config.depth.is_empty(), "depth must not be empty");
        let dim_conv_stem = config.dim_conv_stem.unwrap_or(config.dim);
        let conv_stem1 = nn::conv2d(
            vs / "conv_stem1",
            config.channels,
            dim_conv_stem,
            3,
            nn::ConvConfig {
                stride: 2,
                padding: 1,
                ..Default::default()
            },
        );
        let conv_stem2 = nn::conv2d(
            vs / "conv_stem2",
            dim_conv_stem,
            dim_conv_stem,
            3,
            nn::ConvConfig {
                padding: 1,
                ..Default::default()
            },
        );
        let mut dims = vec![dim_conv_stem];
        dims.extend((0..config.depth.len()).map(|index| config.dim * 2_i64.pow(index as u32)));
        let mut layers = Vec::new();

        for stage_index in 0..config.depth.len() {
            let dim_in = dims[stage_index];
            let dim_out = dims[stage_index + 1];

            for layer_index in 0..config.depth[stage_index] {
                let name = format!("stage_{stage_index}_layer_{layer_index}");
                let current_dim_in = if layer_index == 0 { dim_in } else { dim_out };
                layers.push(MaxViTLayer::new(
                    &(vs / name.as_str()),
                    current_dim_in,
                    dim_out,
                    layer_index == 0,
                    &config,
                ));
            }
        }

        let final_dim = *dims.last().expect("final dim");
        let head_norm = nn::layer_norm(vs / "head_norm", vec![final_dim], Default::default());
        let head = nn::linear(
            vs / "head",
            final_dim,
            config.num_classes,
            Default::default(),
        );

        Self {
            conv_stem1,
            conv_stem2,
            layers,
            head_norm,
            head,
        }
    }
}

impl nn::ModuleT for MaxViT {
    fn forward_t(&self, xs: &Tensor, train: bool) -> Tensor {
        let mut xs = xs.apply(&self.conv_stem1).apply(&self.conv_stem2);

        for layer in &self.layers {
            xs = layer.forward_t(&xs, train);
        }

        xs.mean_dim(&[2_i64, 3][..], false, Kind::Float)
            .apply(&self.head_norm)
            .apply(&self.head)
    }
}