libmir-metal 0.2.0

Metal inference backend for libmir
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125

use super::{Kernels, template};
use crate::engine::{Array, Error, Result, Stream};

#[derive(Debug, Clone, Copy)]
pub struct MxFp4Shape {
    pub tokens: usize,
    pub top_k: usize,
    pub hidden: usize,
    pub intermediate: usize,
}

impl Kernels {
    pub(crate) fn mxfp4_gate_up(
        &self,
        inputs: [&Array; 6],
        shape: MxFp4Shape,
        stream: &Stream,
    ) -> Result<Array> {
        validate(shape)?;
        let output = mirtal::OutputSpec::new(
            mirtal::Shape::new([shape.tokens, shape.top_k, shape.intermediate])?,
            inputs[0].native().dtype()?,
        );
        let [output] = self.mxfp4_gate_up.dispatch(
            stream.native(),
            inputs.map(Array::native),
            &[output],
            &mirtal::Dispatch::new(
                [shape.intermediate * 32, shape.top_k, shape.tokens],
                [32, 1, 1],
            )
            .templates(templates(shape, inputs[0].native().dtype()?)?),
        )?;
        Array::from_native(output)
    }

    pub(crate) fn mxfp4_down(
        &self,
        inputs: [&Array; 6],
        shape: MxFp4Shape,
        stream: &Stream,
    ) -> Result<Array> {
        validate(shape)?;
        let output = mirtal::OutputSpec::new(
            mirtal::Shape::new([shape.tokens, shape.hidden])?,
            inputs[0].native().dtype()?,
        );
        let [output] = self.mxfp4_down.dispatch(
            stream.native(),
            inputs.map(Array::native),
            &[output],
            &mirtal::Dispatch::new([shape.hidden * 32, shape.tokens, 1], [32, 1, 1])
                .templates(templates(shape, inputs[0].native().dtype()?)?),
        )?;
        Array::from_native(output)
    }

    pub(crate) fn mxfp4_split_gate_up(
        &self,
        inputs: [&Array; 9],
        shape: MxFp4Shape,
        stream: &Stream,
    ) -> Result<Array> {
        validate(shape)?;
        let output = mirtal::OutputSpec::new(
            mirtal::Shape::new([shape.tokens, shape.top_k, shape.intermediate])?,
            inputs[0].native().dtype()?,
        );
        let [output] = self.mxfp4_split_gate_up.dispatch(
            stream.native(),
            inputs.map(Array::native),
            &[output],
            &mirtal::Dispatch::new(
                [shape.intermediate * 32, shape.top_k, shape.tokens],
                [32, 1, 1],
            )
            .templates(templates(shape, inputs[0].native().dtype()?)?),
        )?;
        Array::from_native(output)
    }

    pub(crate) fn mxfp4_u32_down(
        &self,
        inputs: [&Array; 6],
        shape: MxFp4Shape,
        stream: &Stream,
    ) -> Result<Array> {
        validate(shape)?;
        let output = mirtal::OutputSpec::new(
            mirtal::Shape::new([shape.tokens, shape.hidden])?,
            inputs[0].native().dtype()?,
        );
        let [output] = self.mxfp4_u32_down.dispatch(
            stream.native(),
            inputs.map(Array::native),
            &[output],
            &mirtal::Dispatch::new([shape.hidden * 32, shape.tokens, 1], [32, 1, 1])
                .templates(templates(shape, inputs[0].native().dtype()?)?),
        )?;
        Array::from_native(output)
    }
}

fn templates(shape: MxFp4Shape, dtype: mirtal::DType) -> Result<[mirtal::TemplateArg; 4]> {
    Ok([
        mirtal::TemplateArg::dtype("T", dtype),
        template("HIDDEN", shape.hidden)?,
        template("INTERMEDIATE", shape.intermediate)?,
        template("TOP_K", shape.top_k)?,
    ])
}

fn validate(shape: MxFp4Shape) -> Result<()> {
    if shape.tokens == 0
        || shape.top_k == 0
        || shape.hidden == 0
        || shape.intermediate == 0
        || !shape.hidden.is_multiple_of(32)
        || !shape.intermediate.is_multiple_of(32)
    {
        Err(Error::InvalidModel("invalid MXFP4 expert geometry".into()))
    } else {
        Ok(())
    }
}