libmir-metal 0.2.0

Metal inference backend for libmir
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93

use models::weights::{TensorBinding, TensorStorage};

use crate::engine::{
    Array, DenseEmbedding, DenseLinear, ModelTensors, QuantizedEmbedding, QuantizedLinear, Result,
    Stream,
};

#[derive(Debug)]
pub(super) enum BoundLinear {
    Dense(DenseLinear),
    Quantized(QuantizedLinear),
}

impl BoundLinear {
    pub(super) fn load_binding(
        tensors: &ModelTensors,
        binding: &TensorBinding,
        stream: &Stream,
    ) -> Result<Self> {
        match &binding.storage {
            TensorStorage::Dense { bias, .. } => {
                DenseLinear::load_names(tensors, &binding.source, bias.as_deref(), None, stream)
                    .map(Self::Dense)
            },
            TensorStorage::AffineQuantized {
                scales,
                biases: Some(biases),
                output_bias,
                group_size: Some(group_size),
                ..
            } => QuantizedLinear::load_names(
                tensors,
                &binding.source,
                scales,
                biases,
                output_bias.as_deref(),
                i32::try_from(*group_size)?,
            )
            .map(Self::Quantized),
            _ => Err(crate::engine::Error::InvalidQuantization(format!(
                "unsupported clamped-routed linear binding {}",
                binding.source
            ))),
        }
    }

    pub(super) fn forward(&self, input: &Array, stream: &Stream) -> Result<Array> {
        match self {
            Self::Dense(linear) => linear.forward(input, stream),
            Self::Quantized(linear) => linear.forward(input, stream),
        }
    }
}

#[derive(Debug)]
pub(super) enum BoundEmbedding {
    Dense(DenseEmbedding),
    Quantized(QuantizedEmbedding),
}

impl BoundEmbedding {
    pub(super) fn load_binding(tensors: &ModelTensors, binding: &TensorBinding) -> Result<Self> {
        match &binding.storage {
            TensorStorage::Dense { .. } => {
                DenseEmbedding::load_name(tensors, &binding.source).map(Self::Dense)
            },
            TensorStorage::AffineQuantized {
                scales,
                biases: Some(biases),
                group_size: Some(group_size),
                ..
            } => QuantizedEmbedding::load_names(
                tensors,
                &binding.source,
                scales,
                biases,
                i32::try_from(*group_size)?,
            )
            .map(Self::Quantized),
            _ => Err(crate::engine::Error::InvalidQuantization(format!(
                "unsupported clamped-routed embedding binding {}",
                binding.source
            ))),
        }
    }

    pub(super) fn lookup(&self, indices: &Array, stream: &Stream) -> Result<Array> {
        match self {
            Self::Dense(embedding) => embedding.lookup(indices, stream),
            Self::Quantized(embedding) => embedding.lookup(indices, stream),
        }
    }
}