vit_tch 0.1.0

Vision Transformer models and building blocks for Rust using tch.
Documentation
use tch::{Kind, Tensor, nn};

pub fn l2_normalize(xs: &Tensor, dim: i64, eps: f64) -> Tensor {
    let denom = xs
        .square()
        .sum_dim_intlist(&[dim][..], true, Kind::Float)
        .clamp_min(eps)
        .sqrt();

    xs / denom
}

#[derive(Debug)]
pub struct RmsNormHeads {
    gamma: Tensor,
    scale: f64,
}

impl RmsNormHeads {
    pub fn new(vs: &nn::Path, heads: i64, dim: i64) -> Self {
        let scale = (dim as f64).sqrt();
        let gamma = vs.var("gamma", &[heads, 1, dim], nn::Init::Const(1.0 / scale));

        Self { gamma, scale }
    }

    pub fn forward(&self, xs: &Tensor) -> Tensor {
        l2_normalize(xs, -1, 1e-12) * self.scale * self.gamma.unsqueeze(0)
    }
}