vit_tch 0.1.0

Vision Transformer models and building blocks for Rust using tch.
Documentation
use tch::{Kind, Tensor, nn};

use crate::{
    config::{CvTConfig, CvTStageConfig},
    conv_layers::{ChannelLayerNorm2D, DepthwiseConv2D},
};

#[derive(Debug)]
struct ConvFeedForward {
    norm: ChannelLayerNorm2D,
    conv1: nn::Conv2D,
    conv2: nn::Conv2D,
    dropout: f64,
}

impl ConvFeedForward {
    fn new(vs: &nn::Path, dim: i64, mult: i64, dropout: f64) -> Self {
        assert!(mult > 0, "mlp multiplier must be positive");
        let norm = ChannelLayerNorm2D::new(&(vs / "norm"), dim, 1e-5);
        let conv1 = nn::conv2d(vs / "conv1", dim, dim * mult, 1, Default::default());
        let conv2 = nn::conv2d(vs / "conv2", dim * mult, dim, 1, Default::default());

        Self {
            norm,
            conv1,
            conv2,
            dropout,
        }
    }
}

impl nn::ModuleT for ConvFeedForward {
    fn forward_t(&self, xs: &Tensor, train: bool) -> Tensor {
        xs.apply(&self.norm)
            .apply(&self.conv1)
            .gelu("none")
            .dropout(self.dropout, train)
            .apply(&self.conv2)
            .dropout(self.dropout, train)
    }
}

#[derive(Debug)]
struct ConvAttention {
    norm: ChannelLayerNorm2D,
    to_q: DepthwiseConv2D,
    to_kv: DepthwiseConv2D,
    to_out: nn::Conv2D,
    heads: i64,
    dim_head: i64,
    scale: f64,
    dropout: f64,
}

impl ConvAttention {
    fn new(
        vs: &nn::Path,
        dim: i64,
        proj_kernel: i64,
        kv_proj_stride: i64,
        heads: i64,
        dim_head: i64,
        dropout: f64,
    ) -> Self {
        assert!(heads > 0, "heads must be positive");
        assert!(dim_head > 0, "dim head must be positive");
        assert!(proj_kernel > 0, "projection kernel must be positive");
        assert!(kv_proj_stride > 0, "kv projection stride must be positive");
        let inner_dim = heads * dim_head;
        let padding = proj_kernel / 2;
        let norm = ChannelLayerNorm2D::new(&(vs / "norm"), dim, 1e-5);
        let to_q = DepthwiseConv2D::new(
            &(vs / "to_q"),
            dim,
            inner_dim,
            proj_kernel,
            padding,
            1,
            false,
        );
        let to_kv = DepthwiseConv2D::new(
            &(vs / "to_kv"),
            dim,
            inner_dim * 2,
            proj_kernel,
            padding,
            kv_proj_stride,
            false,
        );
        let to_out = nn::conv2d(vs / "to_out", inner_dim, dim, 1, Default::default());

        Self {
            norm,
            to_q,
            to_kv,
            to_out,
            heads,
            dim_head,
            scale: (dim_head as f64).powf(-0.5),
            dropout,
        }
    }
}

impl nn::ModuleT for ConvAttention {
    fn forward_t(&self, xs: &Tensor, train: bool) -> Tensor {
        let size = xs.size();
        let batch = size[0];
        let height = size[2];
        let width = size[3];
        let xs = xs.apply(&self.norm);
        let q = xs.apply(&self.to_q);
        let kv = xs.apply(&self.to_kv).chunk(2, 1);
        let q = q
            .view([batch, self.heads, self.dim_head, height * width])
            .transpose(2, 3);
        let k_size = kv[0].size();
        let kv_tokens = k_size[2] * k_size[3];
        let k = kv[0]
            .view([batch, self.heads, self.dim_head, kv_tokens])
            .transpose(2, 3);
        let v = kv[1]
            .view([batch, self.heads, self.dim_head, kv_tokens])
            .transpose(2, 3);
        let attn = (q.matmul(&k.transpose(-1, -2)) * self.scale)
            .softmax(-1, Kind::Float)
            .dropout(self.dropout, train);

        attn.matmul(&v)
            .transpose(2, 3)
            .contiguous()
            .view([batch, self.heads * self.dim_head, height, width])
            .apply(&self.to_out)
            .dropout(self.dropout, train)
    }
}

#[derive(Debug)]
struct ConvTransformerLayer {
    attention: ConvAttention,
    feed_forward: ConvFeedForward,
}

impl ConvTransformerLayer {
    fn new(vs: &nn::Path, dim: i64, stage: CvTStageConfig, dim_head: i64, dropout: f64) -> Self {
        let attention = ConvAttention::new(
            &(vs / "attention"),
            dim,
            stage.proj_kernel,
            stage.kv_proj_stride,
            stage.heads,
            dim_head,
            dropout,
        );
        let feed_forward =
            ConvFeedForward::new(&(vs / "feed_forward"), dim, stage.mlp_mult, dropout);

        Self {
            attention,
            feed_forward,
        }
    }
}

impl nn::ModuleT for ConvTransformerLayer {
    fn forward_t(&self, xs: &Tensor, train: bool) -> Tensor {
        let xs = xs + self.attention.forward_t(xs, train);
        let ff_out = self.feed_forward.forward_t(&xs, train);

        xs + ff_out
    }
}

#[derive(Debug)]
struct CvTStage {
    embedding: nn::Conv2D,
    norm: ChannelLayerNorm2D,
    layers: Vec<ConvTransformerLayer>,
}

impl CvTStage {
    fn new(vs: &nn::Path, dim_in: i64, stage: CvTStageConfig, dim_head: i64, dropout: f64) -> Self {
        assert!(stage.embed_kernel > 0, "embedding kernel must be positive");
        assert!(stage.embed_stride > 0, "embedding stride must be positive");
        let embedding = nn::conv2d(
            vs / "embedding",
            dim_in,
            stage.embed_dim,
            stage.embed_kernel,
            nn::ConvConfig {
                stride: stage.embed_stride,
                padding: stage.embed_kernel / 2,
                ..Default::default()
            },
        );
        let norm = ChannelLayerNorm2D::new(&(vs / "norm"), stage.embed_dim, 1e-5);
        let layers = (0..stage.depth)
            .map(|index| {
                let name = format!("layer_{index}");
                ConvTransformerLayer::new(
                    &(vs / name.as_str()),
                    stage.embed_dim,
                    stage,
                    dim_head,
                    dropout,
                )
            })
            .collect();

        Self {
            embedding,
            norm,
            layers,
        }
    }
}

impl nn::ModuleT for CvTStage {
    fn forward_t(&self, xs: &Tensor, train: bool) -> Tensor {
        let mut xs = xs.apply(&self.embedding).apply(&self.norm);

        for layer in &self.layers {
            xs = layer.forward_t(&xs, train);
        }

        xs
    }
}

#[derive(Debug)]
pub struct CvT {
    stages: Vec<CvTStage>,
    head: nn::Linear,
}

impl CvT {
    pub fn new(vs: &nn::Path, config: CvTConfig) -> Self {
        let mut dim_in = config.channels;
        let stages = config
            .stages
            .iter()
            .enumerate()
            .map(|(index, stage)| {
                let name = format!("stage_{index}");
                let built = CvTStage::new(
                    &(vs / name.as_str()),
                    dim_in,
                    *stage,
                    config.dim_head,
                    config.dropout,
                );
                dim_in = stage.embed_dim;

                built
            })
            .collect();
        let head = nn::linear(vs / "head", dim_in, config.num_classes, Default::default());

        Self { stages, head }
    }
}

impl nn::ModuleT for CvT {
    fn forward_t(&self, xs: &Tensor, train: bool) -> Tensor {
        let mut xs = xs.shallow_clone();

        for stage in &self.stages {
            xs = stage.forward_t(&xs, train);
        }

        xs.adaptive_avg_pool2d([1, 1]).flat_view().apply(&self.head)
    }
}