vit_tch 0.1.0

Vision Transformer models and building blocks for Rust using tch.
Documentation
use tch::{Kind, Tensor, nn};

#[derive(Debug, Clone, Copy)]
pub struct PatchDropout {
    prob: f64,
}

impl PatchDropout {
    pub fn new(prob: f64) -> Self {
        assert!((0.0..1.0).contains(&prob));
        Self { prob }
    }
}

impl nn::ModuleT for PatchDropout {
    fn forward_t(&self, xs: &Tensor, train: bool) -> Tensor {
        if !train || self.prob == 0.0 {
            return xs.shallow_clone();
        }

        let size = xs.size();
        assert_eq!(
            size.len(),
            3,
            "expected token tensor with shape [batch, tokens, dim]"
        );
        let batch = size[0];
        let tokens = size[1];
        let dim = size[2];
        let keep = 1.max((tokens as f64 * (1.0 - self.prob)) as i64);
        let indices = Tensor::rand([batch, tokens], (Kind::Float, xs.device()))
            .topk(keep, -1, true, false)
            .1
            .unsqueeze(-1)
            .repeat([1, 1, dim]);

        xs.gather(1, &indices, false)
    }
}