libmir-metal 0.3.0

Metal inference backend for libmir
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58

use super::{Kernels, template};
use crate::engine::{Error, Result};

mirtal::metal_kernel! {
    fn expert_group {
        name: "mirmir_expert_group",
        templates: [ROUTES: int = 64, EXPERTS: int = 8],
        inputs: [indices: u32],
        outputs: [order: u32, inverse: u32],
        source: file "kernels/expert_group.metal",
        header: inline "",
        row_contiguous: true,
        atomic_outputs: false,
    }
}

#[derive(Debug)]
pub(super) struct ExpertGroupKernel {
    kernel: mirtal::MetalKernel<1, 2>,
}

impl ExpertGroupKernel {
    pub(super) fn new() -> Result<Self> {
        Ok(Self { kernel: expert_group()? })
    }

    pub(super) fn forward(
        &self,
        stream: &mirtal::Stream,
        indices: &mirtal::Array,
        experts: usize,
    ) -> Result<[mirtal::Array; 2]> {
        let routes = indices.len();
        if indices.dtype()? != mirtal::DType::Uint32 || experts == 0 || experts > 1024 {
            return Err(Error::InvalidModel("expert grouping configuration is unsupported".into()));
        }
        let output = mirtal::OutputSpec::new(mirtal::Shape::new([routes])?, mirtal::DType::Uint32);
        let threads = experts.max(routes.min(256));
        Ok(self.kernel.dispatch(
            stream,
            [indices],
            &[output.clone(), output],
            &mirtal::Dispatch::new([threads, 1, 1], [threads, 1, 1])
                .templates([template("ROUTES", routes)?, template("EXPERTS", experts)?]),
        )?)
    }
}

impl Kernels {
    pub(crate) fn expert_group(
        &self,
        stream: &mirtal::Stream,
        indices: &mirtal::Array,
        experts: usize,
    ) -> Result<[mirtal::Array; 2]> {
        self.expert_group.forward(stream, indices, experts)
    }
}