vit_tch 0.1.0

Vision Transformer models and building blocks for Rust using tch.
Documentation
use tch::{IndexOp, Tensor, nn};

use crate::{
    config::{Pool, ViT3DConfig},
    layers::Transformer,
    tensor::{assert_video_patchable, patchify_3d_flat, repeat_token},
};

#[derive(Debug)]
pub struct PatchEmbedding3D {
    norm1: nn::LayerNorm,
    linear: nn::Linear,
    norm2: nn::LayerNorm,
    frame_patch_size: i64,
    patch_height: i64,
    patch_width: i64,
    flatten: bool,
}

impl PatchEmbedding3D {
    pub fn new(
        vs: &nn::Path,
        patch_dim: i64,
        dim: i64,
        frame_patch_size: i64,
        patch_height: i64,
        patch_width: i64,
        flatten: bool,
    ) -> Self {
        let norm1 = nn::layer_norm(vs / "norm1", vec![patch_dim], Default::default());
        let linear = nn::linear(vs / "linear", patch_dim, dim, Default::default());
        let norm2 = nn::layer_norm(vs / "norm2", vec![dim], Default::default());

        Self {
            norm1,
            linear,
            norm2,
            frame_patch_size,
            patch_height,
            patch_width,
            flatten,
        }
    }
}

impl nn::Module for PatchEmbedding3D {
    fn forward(&self, xs: &Tensor) -> Tensor {
        let xs = if self.flatten {
            patchify_3d_flat(
                xs,
                self.frame_patch_size,
                self.patch_height,
                self.patch_width,
            )
        } else {
            crate::tensor::patchify_3d_grid(
                xs,
                self.frame_patch_size,
                self.patch_height,
                self.patch_width,
            )
        };

        xs.apply(&self.norm1).apply(&self.linear).apply(&self.norm2)
    }
}

#[derive(Debug)]
pub struct ViT3D {
    patch_embedding: PatchEmbedding3D,
    cls_token: Tensor,
    pos_embedding: Tensor,
    transformer: Transformer,
    mlp_norm: nn::LayerNorm,
    mlp_head: nn::Linear,
    pool: Pool,
    emb_dropout: f64,
}

impl ViT3D {
    pub fn new(vs: &nn::Path, config: ViT3DConfig) -> Self {
        let image_height = config.image_size.height;
        let image_width = config.image_size.width;
        let patch_height = config.image_patch_size.height;
        let patch_width = config.image_patch_size.width;
        assert_video_patchable(
            config.frames,
            config.frame_patch_size,
            image_height,
            image_width,
            patch_height,
            patch_width,
        );
        let frame_tokens = config.frames / config.frame_patch_size;
        let height_tokens = image_height / patch_height;
        let width_tokens = image_width / patch_width;
        let num_patches = frame_tokens * height_tokens * width_tokens;
        let patch_dim = config.channels * config.frame_patch_size * patch_height * patch_width;
        let patch_embedding = PatchEmbedding3D::new(
            &(vs / "patch_embedding"),
            patch_dim,
            config.dim,
            config.frame_patch_size,
            patch_height,
            patch_width,
            true,
        );
        let pos_embedding = vs.var(
            "pos_embedding",
            &[1, num_patches + 1, config.dim],
            nn::Init::Randn {
                mean: 0.0,
                stdev: 1.0,
            },
        );
        let cls_token = vs.var(
            "cls_token",
            &[1, 1, config.dim],
            nn::Init::Randn {
                mean: 0.0,
                stdev: 1.0,
            },
        );
        let transformer = Transformer::new(
            &(vs / "transformer"),
            config.dim,
            config.depth,
            config.heads,
            config.dim_head,
            config.mlp_dim,
            config.dropout,
        );
        let mlp_norm = nn::layer_norm(vs / "mlp_norm", vec![config.dim], Default::default());
        let mlp_head = nn::linear(
            vs / "mlp_head",
            config.dim,
            config.num_classes,
            Default::default(),
        );

        Self {
            patch_embedding,
            cls_token,
            pos_embedding,
            transformer,
            mlp_norm,
            mlp_head,
            pool: config.pool,
            emb_dropout: config.emb_dropout,
        }
    }
}

impl nn::ModuleT for ViT3D {
    fn forward_t(&self, xs: &Tensor, train: bool) -> Tensor {
        let mut xs = xs.apply(&self.patch_embedding);
        let batch = xs.size()[0];
        let tokens = xs.size()[1];
        let cls_tokens = repeat_token(&self.cls_token, batch);
        xs = Tensor::cat(&[cls_tokens, xs], 1);
        xs = (xs + self.pos_embedding.i((.., 0..tokens + 1, ..))).dropout(self.emb_dropout, train);
        let xs = self.transformer.forward_t(&xs, train);
        let xs = match self.pool {
            Pool::Mean => xs.mean_dim(1, false, xs.kind()),
            Pool::Cls => xs.i((.., 0)),
        };

        xs.apply(&self.mlp_norm).apply(&self.mlp_head)
    }
}