vit_tch 0.1.0

Vision Transformer models and building blocks for Rust using tch.
Documentation
use tch::{Kind, Tensor, nn, nn::ModuleT};

use crate::{
    config::{CompactVisionConfig, VATConfig},
    models::compact::CompactImageTransformer,
};

#[derive(Debug)]
pub struct VAT {
    visual: CompactImageTransformer,
    action_proj: nn::Linear,
    head: nn::Linear,
}

impl VAT {
    pub fn new(vs: &nn::Path, config: VATConfig) -> Self {
        let visual_config = CompactVisionConfig {
            image_size: config.image_size,
            patch_size: config.patch_size,
            num_classes: config.dim,
            dim: config.dim,
            depth: config.depth,
            heads: config.heads,
            mlp_dim: config.mlp_dim,
            channels: config.channels,
            dim_head: config.dim_head,
            dropout: config.dropout,
            ..Default::default()
        };
        let action_proj = nn::linear(
            vs / "action_proj",
            config.action_dim,
            config.dim,
            Default::default(),
        );
        let head = nn::linear(
            vs / "head",
            config.dim,
            config.num_classes,
            Default::default(),
        );

        Self {
            visual: CompactImageTransformer::new(&(vs / "visual"), visual_config),
            action_proj,
            head,
        }
    }

    pub fn forward_t_with_actions(&self, video: &Tensor, actions: &Tensor, train: bool) -> Tensor {
        let size = video.size();
        assert_eq!(
            size.len(),
            5,
            "expected video tensor with shape [batch, channels, frames, height, width]"
        );
        let batch = size[0];
        let channels = size[1];
        let frames = size[2];
        let height = size[3];
        let width = size[4];
        let frames_as_batch = video.permute([0, 2, 1, 3, 4]).contiguous().view([
            batch * frames,
            channels,
            height,
            width,
        ]);
        let visual = self
            .visual
            .forward_t(&frames_as_batch, train)
            .view([batch, frames, -1]);
        let action = actions.apply(&self.action_proj);
        let fused = (visual + action).mean_dim(1, false, Kind::Float);

        fused.apply(&self.head)
    }
}

impl nn::ModuleT for VAT {
    fn forward_t(&self, xs: &Tensor, train: bool) -> Tensor {
        let batch = xs.size()[0];
        let frames = xs.size()[2];
        let actions = Tensor::zeros(
            [batch, frames, self.action_proj.ws.size()[1]],
            (xs.kind(), xs.device()),
        );

        self.forward_t_with_actions(xs, &actions, train)
    }
}