vit_tch 0.1.0

Vision Transformer models and building blocks for Rust using tch.
Documentation
use tch::{Kind, Tensor, nn, nn::ModuleT};

use crate::{config::DistillConfig, models::compact::CompactImageTransformer};

#[derive(Debug)]
pub struct Distill {
    student: CompactImageTransformer,
    distill_head: nn::Linear,
    temperature: f64,
}

impl Distill {
    pub fn new(vs: &nn::Path, config: DistillConfig) -> Self {
        let distill_head = nn::linear(
            vs / "distill_head",
            config.student.num_classes,
            config.distill_dim,
            Default::default(),
        );

        Self {
            student: CompactImageTransformer::new(&(vs / "student"), config.student),
            distill_head,
            temperature: config.temperature,
        }
    }

    pub fn forward_t_with_teacher(&self, xs: &Tensor, teacher: &Tensor, train: bool) -> Tensor {
        let logits = self.forward_t(xs, train).apply(&self.distill_head);
        let student_log_prob = (logits / self.temperature).log_softmax(-1, Kind::Float);
        let teacher_prob = (teacher / self.temperature).softmax(-1, Kind::Float);

        -(teacher_prob * student_log_prob)
            .sum_dim_intlist(&[-1_i64][..], false, Kind::Float)
            .mean(Kind::Float)
    }
}

impl nn::ModuleT for Distill {
    fn forward_t(&self, xs: &Tensor, train: bool) -> Tensor {
        self.student.forward_t(xs, train)
    }
}