libmir-metal 0.2.0

Metal inference backend for libmir
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67

use super::{attention::VisionAttention, mlp::VisionMlp};
use crate::engine::{Array, ModelTensors, NormWeight, Result, Stream};

#[derive(Debug)]
pub(super) struct EncoderLayer {
    input_norm: NormWeight,
    attention: VisionAttention,
    post_attention_norm: NormWeight,
    pre_feedforward_norm: NormWeight,
    mlp: VisionMlp,
    post_feedforward_norm: NormWeight,
    eps: f32,
}

impl EncoderLayer {
    #[allow(clippy::too_many_arguments)]
    pub(super) fn load(
        tensors: &ModelTensors,
        prefix: &str,
        query_heads: usize,
        key_value_heads: usize,
        head_dim: usize,
        rope_theta: f64,
        eps: f32,
        stream: &Stream,
    ) -> Result<Self> {
        Ok(Self {
            input_norm: norm(tensors, prefix, "input_layernorm")?,
            attention: VisionAttention::load(
                tensors,
                &format!("{prefix}.self_attn"),
                query_heads,
                key_value_heads,
                head_dim,
                rope_theta,
                eps,
                stream,
            )?,
            post_attention_norm: norm(tensors, prefix, "post_attention_layernorm")?,
            pre_feedforward_norm: norm(tensors, prefix, "pre_feedforward_layernorm")?,
            mlp: VisionMlp::load(tensors, &format!("{prefix}.mlp"), stream)?,
            post_feedforward_norm: norm(tensors, prefix, "post_feedforward_layernorm")?,
            eps,
        })
    }

    pub(super) fn forward(
        &self,
        input: &Array,
        positions: &Array,
        stream: &Stream,
    ) -> Result<Array> {
        let attention_input = self.input_norm.apply(input, self.eps, stream)?;
        let attention = self.attention.forward(&attention_input, positions, stream)?;
        let attention = self.post_attention_norm.apply(&attention, self.eps, stream)?;
        let hidden = input.add(&attention, stream)?;

        let mlp_input = self.pre_feedforward_norm.apply(&hidden, self.eps, stream)?;
        let mlp = self.mlp.forward(&mlp_input, stream)?;
        let mlp = self.post_feedforward_norm.apply(&mlp, self.eps, stream)?;
        hidden.add(&mlp, stream)
    }
}

fn norm(tensors: &ModelTensors, prefix: &str, name: &str) -> Result<NormWeight> {
    NormWeight::load(tensors, &format!("{prefix}.{name}"))
}