vit_tch 0.1.0

Vision Transformer models and building blocks for Rust using tch.
Documentation
use tch::{Kind, Tensor, nn};

use crate::{config::AcceptVideoWrapperConfig, models::compact::CompactImageTransformer};

#[derive(Debug)]
pub struct AcceptVideoWrapper {
    image_model: CompactImageTransformer,
    time_pos_emb: Option<Tensor>,
    embed_proj: Option<nn::Linear>,
    frames: i64,
}

impl AcceptVideoWrapper {
    pub fn new(vs: &nn::Path, config: AcceptVideoWrapperConfig) -> Self {
        assert!(config.frames > 0, "frames must be positive");
        let image_dim = config.image_model.num_classes;
        let time_pos_emb = config.add_time_pos_emb.then(|| {
            vs.var(
                "time_pos_emb",
                &[config.frames, config.proj_embed_to_dim.unwrap_or(image_dim)],
                nn::Init::Randn {
                    mean: 0.0,
                    stdev: 1e-2,
                },
            )
        });
        let embed_proj = config
            .proj_embed_to_dim
            .map(|dim| nn::linear(vs / "embed_proj", image_dim, dim, Default::default()));

        Self {
            image_model: CompactImageTransformer::new(&(vs / "image_model"), config.image_model),
            time_pos_emb,
            embed_proj,
            frames: config.frames,
        }
    }
}

impl nn::ModuleT for AcceptVideoWrapper {
    fn forward_t(&self, xs: &Tensor, train: bool) -> Tensor {
        let size = xs.size();
        assert_eq!(
            size.len(),
            5,
            "expected video tensor with shape [batch, channels, frames, height, width]"
        );
        let batch = size[0];
        let channels = size[1];
        let frames = size[2];
        let height = size[3];
        let width = size[4];
        assert!(
            frames <= self.frames,
            "received more frames than configured time sequence length"
        );
        let frames_as_batch = xs.permute([0, 2, 1, 3, 4]).contiguous().view([
            batch * frames,
            channels,
            height,
            width,
        ]);
        let mut logits = self.image_model.forward_t(&frames_as_batch, train);
        if let Some(embed_proj) = &self.embed_proj {
            logits = logits.apply(embed_proj);
        }
        let dim = logits.size()[1];
        let mut logits = logits.view([batch, frames, dim]);
        if let Some(time_pos_emb) = &self.time_pos_emb {
            logits += time_pos_emb.narrow(0, 0, frames).unsqueeze(0);
        }

        logits.mean_dim(1, false, Kind::Float)
    }
}