vit_tch 0.1.0

Vision Transformer models and building blocks for Rust using tch.
Documentation
use tch::{Kind, Tensor, nn};

#[derive(Debug)]
pub struct ChannelLayerNorm2D {
    gamma: Tensor,
    beta: Tensor,
    eps: f64,
}

impl ChannelLayerNorm2D {
    pub fn new(vs: &nn::Path, dim: i64, eps: f64) -> Self {
        let gamma = vs.ones("gamma", &[1, dim, 1, 1]);
        let beta = vs.zeros("beta", &[1, dim, 1, 1]);

        Self { gamma, beta, eps }
    }
}

impl nn::Module for ChannelLayerNorm2D {
    fn forward(&self, xs: &Tensor) -> Tensor {
        let mean = xs.mean_dim(1, true, Kind::Float);
        let var = xs.var_dim(&[1_i64][..], false, true);

        (xs - mean) / (var + self.eps).sqrt() * &self.gamma + &self.beta
    }
}

#[derive(Debug)]
pub struct DepthwiseConv2D {
    depthwise: nn::Conv2D,
    pointwise: nn::Conv2D,
}

impl DepthwiseConv2D {
    pub fn new(
        vs: &nn::Path,
        dim_in: i64,
        dim_out: i64,
        kernel_size: i64,
        padding: i64,
        stride: i64,
        bias: bool,
    ) -> Self {
        let depthwise = nn::conv2d(
            vs / "depthwise",
            dim_in,
            dim_in,
            kernel_size,
            nn::ConvConfig {
                padding,
                stride,
                groups: dim_in,
                bias,
                ..Default::default()
            },
        );
        let pointwise = nn::conv2d(
            vs / "pointwise",
            dim_in,
            dim_out,
            1,
            nn::ConvConfig {
                bias,
                ..Default::default()
            },
        );

        Self {
            depthwise,
            pointwise,
        }
    }
}

impl nn::Module for DepthwiseConv2D {
    fn forward(&self, xs: &Tensor) -> Tensor {
        xs.apply(&self.depthwise).apply(&self.pointwise)
    }
}

#[derive(Debug)]
pub struct ConvBnSiLU {
    conv: nn::Conv2D,
    bn: nn::BatchNorm,
}

impl ConvBnSiLU {
    pub fn new(
        vs: &nn::Path,
        dim_in: i64,
        dim_out: i64,
        kernel_size: i64,
        stride: i64,
        padding: i64,
        groups: i64,
    ) -> Self {
        assert!(kernel_size > 0, "kernel size must be positive");
        assert!(stride > 0, "stride must be positive");
        assert!(groups > 0, "groups must be positive");
        let conv = nn::conv2d(
            vs / "conv",
            dim_in,
            dim_out,
            kernel_size,
            nn::ConvConfig {
                stride,
                padding,
                groups,
                bias: false,
                ..Default::default()
            },
        );
        let bn = nn::batch_norm2d(vs / "bn", dim_out, Default::default());

        Self { conv, bn }
    }
}

impl nn::ModuleT for ConvBnSiLU {
    fn forward_t(&self, xs: &Tensor, train: bool) -> Tensor {
        xs.apply(&self.conv).apply_t(&self.bn, train).silu()
    }
}

#[derive(Debug)]
pub struct SqueezeExcite {
    fc1: nn::Conv2D,
    fc2: nn::Conv2D,
}

impl SqueezeExcite {
    pub fn new(vs: &nn::Path, dim: i64, hidden_dim: i64) -> Self {
        assert!(hidden_dim > 0, "hidden dimension must be positive");
        let fc1 = nn::conv2d(vs / "fc1", dim, hidden_dim, 1, Default::default());
        let fc2 = nn::conv2d(vs / "fc2", hidden_dim, dim, 1, Default::default());

        Self { fc1, fc2 }
    }
}

impl nn::Module for SqueezeExcite {
    fn forward(&self, xs: &Tensor) -> Tensor {
        let gate = xs
            .adaptive_avg_pool2d([1, 1])
            .apply(&self.fc1)
            .silu()
            .apply(&self.fc2)
            .sigmoid();

        xs * gate
    }
}

#[derive(Debug)]
pub struct MbConv {
    expand: Option<ConvBnSiLU>,
    depthwise: ConvBnSiLU,
    squeeze_excite: SqueezeExcite,
    project_conv: nn::Conv2D,
    project_bn: nn::BatchNorm,
    residual: bool,
    drop_prob: f64,
}

impl MbConv {
    pub fn new(
        vs: &nn::Path,
        dim_in: i64,
        dim_out: i64,
        stride: i64,
        expansion: i64,
        shrinkage: i64,
        drop_prob: f64,
    ) -> Self {
        assert!(expansion > 0, "expansion must be positive");
        assert!(shrinkage > 0, "shrinkage must be positive");
        let hidden_dim = dim_in * expansion;
        let expand = if expansion == 1 {
            None
        } else {
            Some(ConvBnSiLU::new(
                &(vs / "expand"),
                dim_in,
                hidden_dim,
                1,
                1,
                0,
                1,
            ))
        };
        let depthwise = ConvBnSiLU::new(
            &(vs / "depthwise"),
            hidden_dim,
            hidden_dim,
            3,
            stride,
            1,
            hidden_dim,
        );
        let squeeze_excite = SqueezeExcite::new(
            &(vs / "squeeze_excite"),
            hidden_dim,
            1.max(dim_in / shrinkage),
        );
        let project_conv = nn::conv2d(
            vs / "project_conv",
            hidden_dim,
            dim_out,
            1,
            nn::ConvConfig {
                bias: false,
                ..Default::default()
            },
        );
        let project_bn = nn::batch_norm2d(vs / "project_bn", dim_out, Default::default());
        let residual = stride == 1 && dim_in == dim_out;

        Self {
            expand,
            depthwise,
            squeeze_excite,
            project_conv,
            project_bn,
            residual,
            drop_prob,
        }
    }
}

impl nn::ModuleT for MbConv {
    fn forward_t(&self, xs: &Tensor, train: bool) -> Tensor {
        let mut out = match &self.expand {
            Some(expand) => expand.forward_t(xs, train),
            None => xs.shallow_clone(),
        };
        out = self.depthwise.forward_t(&out, train);
        out = out.apply(&self.squeeze_excite);
        out = out
            .apply(&self.project_conv)
            .apply_t(&self.project_bn, train);
        out = crate::layers::drop_path(&out, self.drop_prob, train);

        if self.residual { xs + out } else { out }
    }
}