vit_tch 0.1.0

Vision Transformer models and building blocks for Rust using tch.
Documentation
use tch::{IndexOp, Tensor, nn};

use crate::{
    config::PiTConfig, conv_layers::DepthwiseConv2D, layers::Transformer, tensor::repeat_token,
};

#[derive(Debug)]
struct OverlapPatchEmbedding {
    linear: nn::Linear,
    patch_size: i64,
    stride: i64,
}

impl OverlapPatchEmbedding {
    fn new(vs: &nn::Path, channels: i64, dim: i64, patch_size: i64) -> Self {
        assert!(patch_size > 1, "patch size must be greater than one");
        let patch_dim = channels * patch_size * patch_size;
        let linear = nn::linear(vs / "linear", patch_dim, dim, Default::default());

        Self {
            linear,
            patch_size,
            stride: patch_size / 2,
        }
    }

    fn output_side(&self, image_size: i64) -> i64 {
        ((image_size - self.patch_size) / self.stride) + 1
    }
}

impl nn::Module for OverlapPatchEmbedding {
    fn forward(&self, xs: &Tensor) -> Tensor {
        xs.im2col(
            [self.patch_size, self.patch_size],
            [1, 1],
            [0, 0],
            [self.stride, self.stride],
        )
        .transpose(1, 2)
        .apply(&self.linear)
    }
}

#[derive(Debug)]
struct PoolTokens {
    downsample: DepthwiseConv2D,
    cls_proj: nn::Linear,
}

impl PoolTokens {
    fn new(vs: &nn::Path, dim: i64) -> Self {
        let downsample = DepthwiseConv2D::new(&(vs / "downsample"), dim, dim * 2, 3, 1, 2, true);
        let cls_proj = nn::linear(vs / "cls_proj", dim, dim * 2, Default::default());

        Self {
            downsample,
            cls_proj,
        }
    }
}

impl nn::Module for PoolTokens {
    fn forward(&self, xs: &Tensor) -> Tensor {
        let cls = xs.i((.., 0..1, ..)).apply(&self.cls_proj);
        let tokens = xs.i((.., 1.., ..));
        let size = tokens.size();
        let batch = size[0];
        let num_tokens = size[1];
        let dim = size[2];
        let side = (num_tokens as f64).sqrt() as i64;
        assert_eq!(
            side * side,
            num_tokens,
            "PiT pooling expects square token grid"
        );
        let tokens = tokens
            .view([batch, side, side, dim])
            .permute([0, 3, 1, 2])
            .apply(&self.downsample);
        let pooled_size = tokens.size();
        let channels = pooled_size[1];
        let height = pooled_size[2];
        let width = pooled_size[3];
        let tokens = tokens
            .view([batch, channels, height * width])
            .transpose(1, 2);

        Tensor::cat(&[cls, tokens], 1)
    }
}

#[derive(Debug)]
struct PiTStage {
    transformer: Transformer,
    pool: Option<PoolTokens>,
}

impl PiTStage {
    fn new(
        vs: &nn::Path,
        dim: i64,
        depth: usize,
        heads: i64,
        dim_head: i64,
        mlp_dim: i64,
        dropout: f64,
        pool: bool,
    ) -> Self {
        let transformer = Transformer::new(
            &(vs / "transformer"),
            dim,
            depth,
            heads,
            dim_head,
            mlp_dim,
            dropout,
        );
        let pool = if pool {
            Some(PoolTokens::new(&(vs / "pool"), dim))
        } else {
            None
        };

        Self { transformer, pool }
    }
}

impl nn::ModuleT for PiTStage {
    fn forward_t(&self, xs: &Tensor, train: bool) -> Tensor {
        let xs = self.transformer.forward_t(xs, train);

        match &self.pool {
            Some(pool) => xs.apply(pool),
            None => xs,
        }
    }
}

#[derive(Debug)]
pub struct PiT {
    patch_embedding: OverlapPatchEmbedding,
    cls_token: Tensor,
    pos_embedding: Tensor,
    stages: Vec<PiTStage>,
    mlp_norm: nn::LayerNorm,
    mlp_head: nn::Linear,
    emb_dropout: f64,
}

impl PiT {
    pub fn new(vs: &nn::Path, config: PiTConfig) -> Self {
        assert_eq!(
            config.depths.len(),
            config.heads.len(),
            "PiT depths and heads must have matching lengths"
        );
        assert!(!config.depths.is_empty(), "PiT requires at least one stage");
        assert_eq!(
            config.image_size % config.patch_size,
            0,
            "image dimensions must be divisible by patch size"
        );
        let patch_embedding = OverlapPatchEmbedding::new(
            &(vs / "patch_embedding"),
            config.channels,
            config.dim,
            config.patch_size,
        );
        let output_side = patch_embedding.output_side(config.image_size);
        let num_patches = output_side * output_side;
        let cls_token = vs.var(
            "cls_token",
            &[1, 1, config.dim],
            nn::Init::Randn {
                mean: 0.0,
                stdev: 1.0,
            },
        );
        let pos_embedding = vs.var(
            "pos_embedding",
            &[1, num_patches + 1, config.dim],
            nn::Init::Randn {
                mean: 0.0,
                stdev: 1.0,
            },
        );
        let mut dim = config.dim;
        let stage_count = config.depths.len();
        let stages = config
            .depths
            .iter()
            .zip(config.heads.iter())
            .enumerate()
            .map(|(index, (depth, heads))| {
                let name = format!("stage_{index}");
                let is_last = index + 1 == stage_count;
                let stage = PiTStage::new(
                    &(vs / name.as_str()),
                    dim,
                    *depth,
                    *heads,
                    config.dim_head,
                    config.mlp_dim,
                    config.dropout,
                    !is_last,
                );
                if !is_last {
                    dim *= 2;
                }

                stage
            })
            .collect();
        let mlp_norm = nn::layer_norm(vs / "mlp_norm", vec![dim], Default::default());
        let mlp_head = nn::linear(vs / "mlp_head", dim, config.num_classes, Default::default());

        Self {
            patch_embedding,
            cls_token,
            pos_embedding,
            stages,
            mlp_norm,
            mlp_head,
            emb_dropout: config.emb_dropout,
        }
    }
}

impl nn::ModuleT for PiT {
    fn forward_t(&self, xs: &Tensor, train: bool) -> Tensor {
        let mut xs = xs.apply(&self.patch_embedding);
        let batch = xs.size()[0];
        let tokens = xs.size()[1];
        let cls_tokens = repeat_token(&self.cls_token, batch);
        xs = Tensor::cat(&[cls_tokens, xs], 1);
        xs = (xs + self.pos_embedding.i((.., 0..tokens + 1, ..))).dropout(self.emb_dropout, train);

        for stage in &self.stages {
            xs = stage.forward_t(&xs, train);
        }

        xs.i((.., 0)).apply(&self.mlp_norm).apply(&self.mlp_head)
    }
}