vit_tch 0.1.0

Vision Transformer models and building blocks for Rust using tch.
Documentation
use tch::{IndexOp, Reduction, Tensor, nn, nn::ModuleT};

use crate::{
    config::{MAEConfig, Pool},
    layers::Transformer,
    masking::{RandomMask, gather_tokens, scatter_tokens},
    models::vit::PatchEmbedding,
    tensor::{num_patches, patch_dim, patchify_2d},
};

#[derive(Debug)]
pub struct MAE {
    patch_embedding: PatchEmbedding,
    encoder_pos_embedding: Tensor,
    encoder: Transformer,
    enc_to_dec: Option<nn::Linear>,
    mask_token: Tensor,
    decoder: Transformer,
    decoder_pos_embedding: nn::Embedding,
    to_pixels: nn::Linear,
    masking_ratio: f64,
    patch_height: i64,
    patch_width: i64,
    pool: Pool,
}

impl MAE {
    pub fn new(vs: &nn::Path, config: MAEConfig) -> Self {
        let encoder_config = config.encoder;
        let image_height = encoder_config.image_size.height;
        let image_width = encoder_config.image_size.width;
        let patch_height = encoder_config.patch_size.height;
        let patch_width = encoder_config.patch_size.width;
        let num_patches = num_patches(image_height, image_width, patch_height, patch_width);
        let patch_dim = patch_dim(encoder_config.channels, patch_height, patch_width);
        let cls_tokens = match encoder_config.pool {
            Pool::Cls => 1,
            Pool::Mean => 0,
        };
        let patch_embedding = PatchEmbedding::new(
            &(vs / "patch_embedding"),
            patch_dim,
            encoder_config.dim,
            patch_height,
            patch_width,
        );
        let encoder_pos_embedding = vs.var(
            "encoder_pos_embedding",
            &[num_patches + cls_tokens, encoder_config.dim],
            nn::Init::Randn {
                mean: 0.0,
                stdev: 1.0,
            },
        );
        let encoder = Transformer::new(
            &(vs / "encoder"),
            encoder_config.dim,
            encoder_config.depth,
            encoder_config.heads,
            encoder_config.dim_head,
            encoder_config.mlp_dim,
            encoder_config.dropout,
        );
        let enc_to_dec = if encoder_config.dim == config.decoder_dim {
            None
        } else {
            Some(nn::linear(
                vs / "enc_to_dec",
                encoder_config.dim,
                config.decoder_dim,
                Default::default(),
            ))
        };
        let mask_token = vs.var(
            "mask_token",
            &[config.decoder_dim],
            nn::Init::Randn {
                mean: 0.0,
                stdev: 1.0,
            },
        );
        let decoder = Transformer::new(
            &(vs / "decoder"),
            config.decoder_dim,
            config.decoder_depth,
            config.decoder_heads,
            config.decoder_dim_head,
            config.decoder_dim * 4,
            0.0,
        );
        let decoder_pos_embedding = nn::embedding(
            vs / "decoder_pos_embedding",
            num_patches,
            config.decoder_dim,
            Default::default(),
        );
        let to_pixels = nn::linear(
            vs / "to_pixels",
            config.decoder_dim,
            patch_dim,
            Default::default(),
        );

        Self {
            patch_embedding,
            encoder_pos_embedding,
            encoder,
            enc_to_dec,
            mask_token,
            decoder,
            decoder_pos_embedding,
            to_pixels,
            masking_ratio: config.masking_ratio,
            patch_height,
            patch_width,
            pool: encoder_config.pool,
        }
    }

    fn encode_tokens(&self, tokens: &Tensor, train: bool) -> Tensor {
        self.encoder.forward_t(tokens, train)
    }
}

impl nn::ModuleT for MAE {
    fn forward_t(&self, xs: &Tensor, train: bool) -> Tensor {
        let patches = patchify_2d(xs, self.patch_height, self.patch_width);
        let batch = patches.size()[0];
        let num_patches = patches.size()[1];
        let tokens = self.patch_embedding.forward_patches(&patches);
        let pos_embedding = match self.pool {
            Pool::Cls => self
                .encoder_pos_embedding
                .i(1..num_patches + 1)
                .unsqueeze(0),
            Pool::Mean => self.encoder_pos_embedding.unsqueeze(0),
        };
        let tokens = tokens + pos_embedding;
        let mask = RandomMask::new(batch, num_patches, self.masking_ratio, xs.device());
        let unmasked_tokens = gather_tokens(&tokens, &mask.unmasked_indices);
        let masked_patches = gather_tokens(&patches, &mask.masked_indices);
        let encoded_tokens = self.encode_tokens(&unmasked_tokens, train);
        let decoder_tokens = match &self.enc_to_dec {
            Some(enc_to_dec) => encoded_tokens.apply(enc_to_dec),
            None => encoded_tokens,
        };
        let unmasked_decoder_tokens =
            decoder_tokens + mask.unmasked_indices.apply(&self.decoder_pos_embedding);
        let num_masked = mask.masked_indices.size()[1];
        let mask_tokens = self
            .mask_token
            .unsqueeze(0)
            .unsqueeze(0)
            .repeat([batch, num_masked, 1])
            + mask.masked_indices.apply(&self.decoder_pos_embedding);
        let decoder_dim = mask_tokens.size()[2];
        let decoder_tokens = scatter_tokens(
            batch,
            num_patches,
            decoder_dim,
            &mask.unmasked_indices,
            &unmasked_decoder_tokens,
        ) + scatter_tokens(
            batch,
            num_patches,
            decoder_dim,
            &mask.masked_indices,
            &mask_tokens,
        );
        let decoded_tokens = self.decoder.forward_t(&decoder_tokens, train);
        let masked_tokens = gather_tokens(&decoded_tokens, &mask.masked_indices);
        let pred_pixel_values = masked_tokens.apply(&self.to_pixels);

        pred_pixel_values.mse_loss(&masked_patches, Reduction::Mean)
    }
}