vit_tch 0.1.0

Vision Transformer models and building blocks for Rust using tch.
Documentation
use tch::{IndexOp, Kind, Tensor, nn};

use crate::{
    config::{Pool, ViTWithDecorrConfig},
    layers::Transformer,
    models::vit::PatchEmbedding,
    tensor::{num_patches, patch_dim, repeat_token},
};

#[derive(Debug, Clone)]
pub struct DecorrelationLoss {
    sample_frac: f64,
    soft_validate_num_sampled: bool,
}

impl DecorrelationLoss {
    pub fn new(sample_frac: f64) -> Self {
        Self::with_soft_validation(sample_frac, false)
    }

    pub fn with_soft_validation(sample_frac: f64, soft_validate_num_sampled: bool) -> Self {
        assert!((0.0..=1.0).contains(&sample_frac));
        Self {
            sample_frac,
            soft_validate_num_sampled,
        }
    }

    pub fn forward(&self, tokens: &Tensor) -> Tensor {
        let mut tokens = tokens.shallow_clone();
        let size = tokens.size();
        let seq_len = size[size.len() - 2];
        let dim = size[size.len() - 1];

        if self.sample_frac < 1.0 {
            let num_sampled = (seq_len as f64 * self.sample_frac) as i64;
            assert!(self.soft_validate_num_sampled || num_sampled >= 2);

            if num_sampled <= 1 {
                return Tensor::zeros([], (tokens.kind(), tokens.device()));
            }

            tokens = sample_tokens(&tokens, num_sampled);
        }

        let dist =
            tokens.transpose(-1, -2).matmul(&tokens) / (tokens.size()[tokens.dim() - 2] as f64);
        let eye = Tensor::eye(dim, (tokens.kind(), tokens.device()));
        let loss = dist.pow_tensor_scalar(2.0) * (Tensor::ones_like(&eye) - eye)
            / ((dim - 1) * dim) as f64;
        loss.sum_dim_intlist(&[-1_i64, -2][..], false, Kind::Float)
            .mean(Kind::Float)
    }
}

fn sample_tokens(tokens: &Tensor, num_sampled: i64) -> Tensor {
    let size = tokens.size();
    let dim = size[size.len() - 1];
    let seq_len = size[size.len() - 2];
    let outer: i64 = size[..size.len() - 2].iter().product();
    let flat = tokens.contiguous().view([outer, seq_len, dim]);
    let indices = Tensor::rand([outer, seq_len], (tokens.kind(), tokens.device()))
        .argsort(-1, false)
        .i((.., 0..num_sampled))
        .unsqueeze(-1)
        .repeat([1, 1, dim]);
    let sampled = flat.gather(1, &indices, false);
    let mut new_size = size[..size.len() - 2].to_vec();
    new_size.push(num_sampled);
    new_size.push(dim);

    sampled.view(new_size.as_slice())
}

#[derive(Debug)]
pub struct ViTWithDecorr {
    patch_embedding: PatchEmbedding,
    cls_token: Tensor,
    pos_embedding: Tensor,
    transformer: Transformer,
    mlp_head: nn::Linear,
    pool: Pool,
    emb_dropout: f64,
    decorr_loss: Option<DecorrelationLoss>,
}

impl ViTWithDecorr {
    pub fn new(vs: &nn::Path, config: ViTWithDecorrConfig) -> Self {
        let vit = config.vit;
        let image_height = vit.image_size.height;
        let image_width = vit.image_size.width;
        let patch_height = vit.patch_size.height;
        let patch_width = vit.patch_size.width;
        let num_patches = num_patches(image_height, image_width, patch_height, patch_width);
        let patch_dim = patch_dim(vit.channels, patch_height, patch_width);
        let patch_embedding = PatchEmbedding::new(
            &(vs / "patch_embedding"),
            patch_dim,
            vit.dim,
            patch_height,
            patch_width,
        );
        let pos_embedding = vs.var(
            "pos_embedding",
            &[1, num_patches + 1, vit.dim],
            nn::Init::Randn {
                mean: 0.0,
                stdev: 1.0,
            },
        );
        let cls_token = vs.var(
            "cls_token",
            &[1, 1, vit.dim],
            nn::Init::Randn {
                mean: 0.0,
                stdev: 1.0,
            },
        );
        let transformer = Transformer::new(
            &(vs / "transformer"),
            vit.dim,
            vit.depth,
            vit.heads,
            vit.dim_head,
            vit.mlp_dim,
            vit.dropout,
        );
        let mlp_head = nn::linear(
            vs / "mlp_head",
            vit.dim,
            vit.num_classes,
            Default::default(),
        );
        let decorr_loss = if config.decorr_sample_frac > 0.0 {
            Some(DecorrelationLoss::new(config.decorr_sample_frac))
        } else {
            None
        };

        Self {
            patch_embedding,
            cls_token,
            pos_embedding,
            transformer,
            mlp_head,
            pool: vit.pool,
            emb_dropout: vit.emb_dropout,
            decorr_loss,
        }
    }

    pub fn forward_t_with_aux(
        &self,
        xs: &Tensor,
        train: bool,
        return_decorr_aux_loss: bool,
    ) -> (Tensor, Tensor) {
        let mut xs = xs.apply(&self.patch_embedding);
        let batch = xs.size()[0];
        let tokens = xs.size()[1];
        let cls_tokens = repeat_token(&self.cls_token, batch);
        xs = Tensor::cat(&[cls_tokens, xs], 1);
        xs = (xs + self.pos_embedding.i((.., 0..tokens + 1, ..))).dropout(self.emb_dropout, train);

        let (xs, normed_layer_inputs) = self.transformer.forward_t_with_normed(&xs, train);
        let aux_loss = if return_decorr_aux_loss {
            match &self.decorr_loss {
                Some(decorr_loss) => decorr_loss.forward(&normed_layer_inputs),
                None => Tensor::zeros([], (xs.kind(), xs.device())),
            }
        } else {
            Tensor::zeros([], (xs.kind(), xs.device()))
        };
        let xs = match self.pool {
            Pool::Mean => xs.mean_dim(1, false, xs.kind()),
            Pool::Cls => xs.i((.., 0)),
        };
        let logits = xs.apply(&self.mlp_head);

        (logits, aux_loss)
    }
}

impl nn::ModuleT for ViTWithDecorr {
    fn forward_t(&self, xs: &Tensor, train: bool) -> Tensor {
        self.forward_t_with_aux(xs, train, train).0
    }
}