vit_tch 0.1.0

Vision Transformer models and building blocks for Rust using tch.
Documentation
use tch::Tensor;

pub fn assert_image_patchable(
    image_height: i64,
    image_width: i64,
    patch_height: i64,
    patch_width: i64,
) {
    assert!(
        image_height % patch_height == 0 && image_width % patch_width == 0,
        "image dimensions must be divisible by patch size"
    );
}

pub fn num_patches(
    image_height: i64,
    image_width: i64,
    patch_height: i64,
    patch_width: i64,
) -> i64 {
    assert_image_patchable(image_height, image_width, patch_height, patch_width);
    (image_height / patch_height) * (image_width / patch_width)
}

pub fn patch_dim(channels: i64, patch_height: i64, patch_width: i64) -> i64 {
    channels * patch_height * patch_width
}

pub fn assert_sequence_patchable(seq_len: i64, patch_size: i64) {
    assert!(
        seq_len % patch_size == 0,
        "sequence length must be divisible by patch size"
    );
}

pub fn assert_video_patchable(
    frames: i64,
    frame_patch_size: i64,
    image_height: i64,
    image_width: i64,
    patch_height: i64,
    patch_width: i64,
) {
    assert_image_patchable(image_height, image_width, patch_height, patch_width);
    assert!(
        frames % frame_patch_size == 0,
        "frames must be divisible by frame patch size"
    );
}

pub fn patchify_2d(xs: &Tensor, patch_height: i64, patch_width: i64) -> Tensor {
    let size = xs.size();
    assert_eq!(
        size.len(),
        4,
        "expected image tensor with shape [batch, channels, height, width]"
    );
    let batch = size[0];
    let channels = size[1];
    let height = size[2];
    let width = size[3];
    assert_image_patchable(height, width, patch_height, patch_width);
    let grid_h = height / patch_height;
    let grid_w = width / patch_width;

    xs.view([batch, channels, grid_h, patch_height, grid_w, patch_width])
        .permute([0, 2, 4, 3, 5, 1])
        .contiguous()
        .view([
            batch,
            grid_h * grid_w,
            patch_height * patch_width * channels,
        ])
}

pub fn patchify_1d(xs: &Tensor, patch_size: i64) -> Tensor {
    let size = xs.size();
    assert_eq!(
        size.len(),
        3,
        "expected sequence tensor with shape [batch, channels, length]"
    );
    let batch = size[0];
    let channels = size[1];
    let seq_len = size[2];
    assert_sequence_patchable(seq_len, patch_size);
    let tokens = seq_len / patch_size;

    xs.view([batch, channels, tokens, patch_size])
        .permute([0, 2, 3, 1])
        .contiguous()
        .view([batch, tokens, patch_size * channels])
}

pub fn patchify_3d_flat(
    xs: &Tensor,
    frame_patch_size: i64,
    patch_height: i64,
    patch_width: i64,
) -> Tensor {
    let patches = patchify_3d_grid(xs, frame_patch_size, patch_height, patch_width);
    let size = patches.size();
    let batch = size[0];
    let frame_tokens = size[1];
    let height_tokens = size[2];
    let width_tokens = size[3];
    let dim = size[4];

    patches.view([batch, frame_tokens * height_tokens * width_tokens, dim])
}

pub fn patchify_3d_grid(
    xs: &Tensor,
    frame_patch_size: i64,
    patch_height: i64,
    patch_width: i64,
) -> Tensor {
    let size = xs.size();
    assert_eq!(
        size.len(),
        5,
        "expected video tensor with shape [batch, channels, frames, height, width]"
    );
    let batch = size[0];
    let channels = size[1];
    let frames = size[2];
    let height = size[3];
    let width = size[4];
    assert_video_patchable(
        frames,
        frame_patch_size,
        height,
        width,
        patch_height,
        patch_width,
    );
    let frame_tokens = frames / frame_patch_size;
    let height_tokens = height / patch_height;
    let width_tokens = width / patch_width;

    xs.view(
        &[
            batch,
            channels,
            frame_tokens,
            frame_patch_size,
            height_tokens,
            patch_height,
            width_tokens,
            patch_width,
        ][..],
    )
    .permute([0, 2, 4, 6, 3, 5, 7, 1])
    .contiguous()
    .view([
        batch,
        frame_tokens,
        height_tokens,
        width_tokens,
        frame_patch_size * patch_height * patch_width * channels,
    ])
}

pub fn split_heads(xs: &Tensor, heads: i64) -> Tensor {
    let size = xs.size();
    assert_eq!(
        size.len(),
        3,
        "expected tensor with shape [batch, tokens, channels]"
    );
    let batch = size[0];
    let tokens = size[1];
    let dim = size[2];
    assert_eq!(dim % heads, 0, "channels must be divisible by heads");
    let dim_head = dim / heads;

    xs.view([batch, tokens, heads, dim_head])
        .permute([0, 2, 1, 3])
}

pub fn merge_heads(xs: &Tensor) -> Tensor {
    let size = xs.size();
    assert_eq!(
        size.len(),
        4,
        "expected tensor with shape [batch, heads, tokens, dim_head]"
    );
    let batch = size[0];
    let heads = size[1];
    let tokens = size[2];
    let dim_head = size[3];

    xs.permute([0, 2, 1, 3])
        .contiguous()
        .view([batch, tokens, heads * dim_head])
}

pub fn repeat_token(xs: &Tensor, batch: i64) -> Tensor {
    let size = xs.size();
    match size.as_slice() {
        [dim] => xs.unsqueeze(0).repeat([batch, 1]).view([batch, 1, *dim]),
        [tokens, dim] => xs
            .unsqueeze(0)
            .repeat([batch, 1, 1])
            .view([batch, *tokens, *dim]),
        [1, tokens, dim] => xs.repeat([batch, 1, 1]).view([batch, *tokens, *dim]),
        _ => panic!("expected token tensor with shape [dim], [tokens, dim], or [1, tokens, dim]"),
    }
}

pub fn image_to_tokens(xs: &Tensor) -> Tensor {
    let size = xs.size();
    assert_eq!(
        size.len(),
        4,
        "expected image tensor with shape [batch, channels, height, width]"
    );
    let batch = size[0];
    let channels = size[1];
    let height = size[2];
    let width = size[3];

    xs.permute([0, 2, 3, 1])
        .contiguous()
        .view([batch, height * width, channels])
}

pub fn tokens_to_image(xs: &Tensor, height: i64, width: i64) -> Tensor {
    let size = xs.size();
    assert_eq!(
        size.len(),
        3,
        "expected token tensor with shape [batch, tokens, channels]"
    );
    let batch = size[0];
    let tokens = size[1];
    let channels = size[2];
    assert_eq!(tokens, height * width, "tokens must match image grid");

    xs.view([batch, height, width, channels])
        .permute([0, 3, 1, 2])
        .contiguous()
}

pub fn window_partition(xs: &Tensor, window_height: i64, window_width: i64) -> Tensor {
    let size = xs.size();
    assert_eq!(
        size.len(),
        4,
        "expected image tensor with shape [batch, channels, height, width]"
    );
    let batch = size[0];
    let channels = size[1];
    let height = size[2];
    let width = size[3];
    assert_image_patchable(height, width, window_height, window_width);
    let grid_h = height / window_height;
    let grid_w = width / window_width;

    xs.view([batch, channels, grid_h, window_height, grid_w, window_width])
        .permute([0, 2, 4, 1, 3, 5])
        .contiguous()
        .view([
            batch * grid_h * grid_w,
            channels,
            window_height,
            window_width,
        ])
}

pub fn window_unpartition(
    windows: &Tensor,
    batch: i64,
    height: i64,
    width: i64,
    window_height: i64,
    window_width: i64,
) -> Tensor {
    let size = windows.size();
    assert_eq!(
        size.len(),
        4,
        "expected window tensor with shape [windows, channels, height, width]"
    );
    let channels = size[1];
    assert_image_patchable(height, width, window_height, window_width);
    let grid_h = height / window_height;
    let grid_w = width / window_width;
    assert_eq!(
        size[0],
        batch * grid_h * grid_w,
        "window count must match batch and image grid"
    );

    windows
        .view([batch, grid_h, grid_w, channels, window_height, window_width])
        .permute([0, 3, 1, 4, 2, 5])
        .contiguous()
        .view([batch, channels, height, width])
}