vit_tch 0.1.0

Vision Transformer models and building blocks for Rust using tch.
Documentation
use tch::{Device, Kind, Tensor};

pub fn posemb_sincos_1d(tokens: i64, dim: i64, temperature: f64, device: Device) -> Tensor {
    assert_eq!(
        dim % 2,
        0,
        "feature dimension must be multiple of 2 for sincos embedding"
    );

    let omega = Tensor::arange(dim / 2, (Kind::Float, device)) / ((dim / 2 - 1) as f64);
    let omega = (omega * temperature.ln()).neg().exp();
    let n = Tensor::arange(tokens, (Kind::Float, device)).unsqueeze(1) * omega.unsqueeze(0);

    Tensor::cat(&[n.sin(), n.cos()], 1)
}

pub fn posemb_sincos_2d(
    height: i64,
    width: i64,
    dim: i64,
    temperature: f64,
    device: Device,
) -> Tensor {
    assert_eq!(
        dim % 4,
        0,
        "feature dimension must be multiple of 4 for sincos embedding"
    );

    let omega = Tensor::arange(dim / 4, (Kind::Float, device)) / ((dim / 4 - 1) as f64);
    let omega = (omega * temperature.ln()).neg().exp();
    let y = Tensor::arange(height, (Kind::Float, device))
        .view([height, 1])
        .repeat([1, width])
        .flatten(0, -1)
        .unsqueeze(1);
    let x = Tensor::arange(width, (Kind::Float, device))
        .view([1, width])
        .repeat([height, 1])
        .flatten(0, -1)
        .unsqueeze(1);
    let x = x * omega.unsqueeze(0);
    let y = y * omega.unsqueeze(0);

    Tensor::cat(&[x.sin(), x.cos(), y.sin(), y.cos()], 1)
}

pub fn posemb_sincos_3d(
    frames: i64,
    height: i64,
    width: i64,
    dim: i64,
    temperature: f64,
    device: Device,
) -> Tensor {
    let fourier_dim = dim / 6;
    assert!(
        fourier_dim > 1,
        "feature dimension is too small for 3d sincos embedding"
    );

    let omega = Tensor::arange(fourier_dim, (Kind::Float, device)) / ((fourier_dim - 1) as f64);
    let omega = (omega * temperature.ln()).neg().exp();
    let z = Tensor::arange(frames, (Kind::Float, device))
        .view([frames, 1, 1])
        .repeat([1, height, width])
        .flatten(0, -1)
        .unsqueeze(1);
    let y = Tensor::arange(height, (Kind::Float, device))
        .view([1, height, 1])
        .repeat([frames, 1, width])
        .flatten(0, -1)
        .unsqueeze(1);
    let x = Tensor::arange(width, (Kind::Float, device))
        .view([1, 1, width])
        .repeat([frames, height, 1])
        .flatten(0, -1)
        .unsqueeze(1);
    let x = x * omega.unsqueeze(0);
    let y = y * omega.unsqueeze(0);
    let z = z * omega.unsqueeze(0);
    let pe = Tensor::cat(&[x.sin(), x.cos(), y.sin(), y.cos(), z.sin(), z.cos()], 1);
    let used_dim = fourier_dim * 6;

    if used_dim == dim {
        pe
    } else {
        pe.constant_pad_nd([0, dim - used_dim])
    }
}