libmir-metal 0.1.0

Metal inference backend for libmir
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96

mod convolution;
mod fallback;
mod update;

use super::{Array, Error, Result, Stream};

#[derive(Clone, Copy)]
pub struct GatedDeltaInputs<'a> {
    pub query: &'a Array,
    pub key: &'a Array,
    pub value: &'a Array,
    pub alpha: &'a Array,
    pub beta: &'a Array,
    pub a_log: &'a Array,
    pub dt_bias: &'a Array,
}

#[derive(Debug, Default)]
pub struct GatedDeltaState {
    value: Option<Array>,
    convolution: Option<Array>,
    offset: usize,
}

impl GatedDeltaState {
    pub fn new() -> Result<Self> {
        Ok(Self::default())
    }

    pub const fn offset(&self) -> Result<usize> {
        Ok(self.offset)
    }

    pub fn reset(&mut self) -> Result<()> {
        self.value = None;
        self.convolution = None;
        self.offset = 0;
        Ok(())
    }

    pub fn snapshot(&self) -> Result<Self> {
        Ok(Self {
            value: clone_array(self.value.as_ref())?,
            convolution: clone_array(self.convolution.as_ref())?,
            offset: self.offset,
        })
    }

    pub fn values(&self) -> Result<Array> {
        clone_array(self.value.as_ref())?
            .ok_or_else(|| Error::InvalidModel("Gated Delta cache has no state".into()))
    }

    pub fn convolve_silu(
        &mut self,
        input: &Array,
        weight: &Array,
        stream: &Stream,
    ) -> Result<Array> {
        convolution::convolve(self, input, weight, stream)
    }

    pub fn update(&mut self, inputs: GatedDeltaInputs<'_>, stream: &Stream) -> Result<Array> {
        update::update(self, inputs, stream)
    }

    pub(crate) fn update_normalized(
        &mut self,
        inputs: GatedDeltaInputs<'_>,
        stream: &Stream,
    ) -> Result<Array> {
        update::decode(self, inputs, true, stream)
    }

    pub(crate) fn update_fused(
        &mut self,
        inputs: GatedDeltaInputs<'_>,
        stream: &Stream,
    ) -> Result<Array> {
        update::decode(self, inputs, false, stream)
    }

    pub(crate) fn compiled_decode_state(&self) -> Option<(&Array, &Array)> {
        self.value.as_ref().zip(self.convolution.as_ref())
    }

    pub(crate) fn commit_compiled_decode(&mut self, value: Array, convolution: Array) {
        self.value = Some(value);
        self.convolution = Some(convolution);
        self.offset += 1;
    }
}

fn clone_array(input: Option<&Array>) -> Result<Option<Array>> {
    input.map(|array| Array::from_native(array.native().clone())).transpose()
}