oxidized-transformers 0.1.1

Transformers library (not functional yet)
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92

use candle_core::Tensor;
use snafu::{ResultExt, Snafu};

use crate::{
    architectures::{BuildEncoderLayer, EncoderLayer},
    error::BoxedError,
    layers::{attention::AttentionMask, transformer::TransformerLayerConfig},
};

/// ALBERT layer group errors.
#[derive(Debug, Snafu)]
enum BuildAlbertLayerGroupError {
    #[snafu(display("Cannot build ALBERT group layer {n}"))]
    BuildTransformerLayer {
        source: crate::error::BoxedError,
        n: usize,
    },
}

/// ALBERT layer group configuration
///
/// A group's layer can in turn consist of multiple layers (though this is
/// rarely used in ALBERT models).
#[derive(Debug)]
pub struct AlbertLayerGroupConfig {
    n_layers_per_group: usize,
    transformer_layer: TransformerLayerConfig,
}

impl AlbertLayerGroupConfig {
    /// Number of layers within a group layer.
    pub fn n_layers_per_group(mut self, n_layers_per_group: usize) -> Self {
        self.n_layers_per_group = n_layers_per_group;
        self
    }

    /// Transformer layer configuration.
    pub fn transformer_layer(mut self, transformer_layer: TransformerLayerConfig) -> Self {
        self.transformer_layer = transformer_layer;
        self
    }
}

impl Default for AlbertLayerGroupConfig {
    fn default() -> Self {
        Self {
            n_layers_per_group: 1,
            transformer_layer: TransformerLayerConfig::default(),
        }
    }
}

impl BuildEncoderLayer for AlbertLayerGroupConfig {
    fn build_encoder_layer(
        &self,
        vb: candle_nn::VarBuilder,
    ) -> Result<Box<dyn crate::architectures::EncoderLayer>, crate::error::BoxedError> {
        let layers = (0..self.n_layers_per_group)
            .map({
                |n| {
                    self.transformer_layer
                        .build_encoder_layer(vb.push_prefix(format!("group_layer_{n}")))
                        .context(BuildTransformerLayerSnafu { n })
                }
            })
            .collect::<Result<Vec<_>, _>>()?;

        Ok(Box::new(AlbertLayerGroup { layers }))
    }
}

pub struct AlbertLayerGroup {
    layers: Vec<Box<dyn EncoderLayer>>,
}

impl EncoderLayer for AlbertLayerGroup {
    fn forward_t(
        &self,
        input: &Tensor,
        mask: &AttentionMask,
        positions: Option<&Tensor>,
        train: bool,
    ) -> Result<Tensor, BoxedError> {
        let mut layer_output = input.clone();

        for layer in &self.layers {
            layer_output = layer.forward_t(&layer_output, mask, positions, train)?;
        }

        Ok(layer_output)
    }
}