vit_tch 0.1.0

Vision Transformer models and building blocks for Rust using tch.
Documentation
use tch::{Tensor, nn};

use crate::{
    config::ViTConfig,
    layers::FeedForward,
    models::vit::PatchEmbedding,
    norm::RmsNormHeads,
    positional::posemb_sincos_2d,
    tensor::{assert_image_patchable, merge_heads, patch_dim, split_heads},
};

#[derive(Debug)]
struct QkNormAttention {
    norm: nn::LayerNorm,
    q_norm: RmsNormHeads,
    k_norm: RmsNormHeads,
    to_qkv: nn::Linear,
    to_out: nn::Linear,
    heads: i64,
}

impl QkNormAttention {
    fn new(vs: &nn::Path, dim: i64, heads: i64, dim_head: i64) -> Self {
        let inner_dim = heads * dim_head;
        let norm = nn::layer_norm(vs / "norm", vec![dim], Default::default());
        let q_norm = RmsNormHeads::new(&(vs / "q_norm"), heads, dim_head);
        let k_norm = RmsNormHeads::new(&(vs / "k_norm"), heads, dim_head);
        let to_qkv = nn::linear(
            vs / "to_qkv",
            dim,
            inner_dim * 3,
            nn::LinearConfig {
                bias: false,
                ..Default::default()
            },
        );
        let to_out = nn::linear(
            vs / "to_out",
            inner_dim,
            dim,
            nn::LinearConfig {
                bias: false,
                ..Default::default()
            },
        );

        Self {
            norm,
            q_norm,
            k_norm,
            to_qkv,
            to_out,
            heads,
        }
    }
}

impl nn::ModuleT for QkNormAttention {
    fn forward_t(&self, xs: &Tensor, _train: bool) -> Tensor {
        let qkv = xs.apply(&self.norm).apply(&self.to_qkv).chunk(3, -1);
        let q = self.q_norm.forward(&split_heads(&qkv[0], self.heads));
        let k = self.k_norm.forward(&split_heads(&qkv[1], self.heads));
        let v = split_heads(&qkv[2], self.heads);
        let attn = q.matmul(&k.transpose(-1, -2)).softmax(-1, q.kind());

        merge_heads(&attn.matmul(&v)).apply(&self.to_out)
    }
}

#[derive(Debug)]
struct QkNormTransformerLayer {
    attention: QkNormAttention,
    feed_forward: FeedForward,
}

impl QkNormTransformerLayer {
    fn new(vs: &nn::Path, dim: i64, heads: i64, dim_head: i64, mlp_dim: i64) -> Self {
        let attention = QkNormAttention::new(&(vs / "attention"), dim, heads, dim_head);
        let feed_forward = FeedForward::new(&(vs / "feed_forward"), dim, mlp_dim, 0.0);

        Self {
            attention,
            feed_forward,
        }
    }
}

impl nn::ModuleT for QkNormTransformerLayer {
    fn forward_t(&self, xs: &Tensor, train: bool) -> Tensor {
        let xs = xs + self.attention.forward_t(xs, train);
        let ff_out = self.feed_forward.forward_t(&xs, train);
        xs + ff_out
    }
}

#[derive(Debug)]
struct QkNormTransformer {
    layers: Vec<QkNormTransformerLayer>,
    norm: nn::LayerNorm,
}

impl QkNormTransformer {
    fn new(vs: &nn::Path, dim: i64, depth: usize, heads: i64, dim_head: i64, mlp_dim: i64) -> Self {
        let layers = (0..depth)
            .map(|index| {
                let name = format!("layer_{index}");
                QkNormTransformerLayer::new(&(vs / name.as_str()), dim, heads, dim_head, mlp_dim)
            })
            .collect();
        let norm = nn::layer_norm(vs / "norm", vec![dim], Default::default());

        Self { layers, norm }
    }
}

impl nn::ModuleT for QkNormTransformer {
    fn forward_t(&self, xs: &Tensor, train: bool) -> Tensor {
        let mut xs = xs.shallow_clone();

        for layer in &self.layers {
            xs = layer.forward_t(&xs, train);
        }

        xs.apply(&self.norm)
    }
}

#[derive(Debug)]
pub struct SimpleViTWithQkNorm {
    patch_embedding: PatchEmbedding,
    pos_embedding: Tensor,
    transformer: QkNormTransformer,
    linear_head: nn::LayerNorm,
}

impl SimpleViTWithQkNorm {
    pub fn new(vs: &nn::Path, config: ViTConfig) -> Self {
        let image_height = config.image_size.height;
        let image_width = config.image_size.width;
        let patch_height = config.patch_size.height;
        let patch_width = config.patch_size.width;
        assert_image_patchable(image_height, image_width, patch_height, patch_width);
        let patch_dim = patch_dim(config.channels, patch_height, patch_width);
        let patch_embedding = PatchEmbedding::new(
            &(vs / "patch_embedding"),
            patch_dim,
            config.dim,
            patch_height,
            patch_width,
        );
        let pos_embedding = posemb_sincos_2d(
            image_height / patch_height,
            image_width / patch_width,
            config.dim,
            10000.0,
            vs.device(),
        );
        let transformer = QkNormTransformer::new(
            &(vs / "transformer"),
            config.dim,
            config.depth,
            config.heads,
            config.dim_head,
            config.mlp_dim,
        );
        let linear_head = nn::layer_norm(vs / "linear_head", vec![config.dim], Default::default());

        Self {
            patch_embedding,
            pos_embedding,
            transformer,
            linear_head,
        }
    }
}

impl nn::ModuleT for SimpleViTWithQkNorm {
    fn forward_t(&self, xs: &Tensor, train: bool) -> Tensor {
        let xs = xs.apply(&self.patch_embedding) + self.pos_embedding.unsqueeze(0);
        let xs = self.transformer.forward_t(&xs, train);
        let xs = xs.mean_dim(1, false, xs.kind());

        xs.apply(&self.linear_head)
    }
}