libmir-cuda 0.3.0

CUDA inference backend for libmir
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128

use mircuda::{DeviceBuffer, bf16};

use super::super::weights::ClampedRoutedExpertWeights;
use crate::{
    CudaTensor, Error, Result, backend::tuning::ClampedMoeExecution, kernels::ClampedRoutedKernels,
};

pub(super) struct Candidate {
    pub(super) execution: ClampedMoeExecution,
    kernels: ClampedRoutedKernels,
}

impl Candidate {
    pub(super) const fn new(kernels: ClampedRoutedKernels, execution: ClampedMoeExecution) -> Self {
        Self { execution, kernels }
    }

    #[allow(clippy::too_many_arguments)]
    pub(super) fn execute(
        &self,
        stream: &mircuda::Stream,
        weights: &ClampedRoutedExpertWeights,
        input: &DeviceBuffer<bf16>,
        selected: &DeviceBuffer<u32>,
        routing: &DeviceBuffer<bf16>,
        activated: &mut DeviceBuffer<bf16>,
        partial: &mut DeviceBuffer<f32>,
        output: &mut DeviceBuffer<bf16>,
    ) -> Result<()> {
        match weights {
            ClampedRoutedExpertWeights::Native(weights) => {
                self.kernels.gate_up_native(
                    stream,
                    input,
                    u8s(&weights.gate_up_blocks)?,
                    u8s(&weights.gate_up_scales)?,
                    bf16s(&weights.gate_up_bias)?,
                    selected,
                    activated,
                )?;
                match self.execution {
                    ClampedMoeExecution::RouteParallel => self.kernels.down_routes_native(
                        stream,
                        activated,
                        u8s(&weights.down_blocks)?,
                        u8s(&weights.down_scales)?,
                        bf16s(&weights.down_bias)?,
                        selected,
                        routing,
                        partial,
                        output,
                    ),
                    ClampedMoeExecution::FusedReduce => self.kernels.down_native(
                        stream,
                        activated,
                        u8s(&weights.down_blocks)?,
                        u8s(&weights.down_scales)?,
                        bf16s(&weights.down_bias)?,
                        selected,
                        routing,
                        output,
                    ),
                }
            },
            ClampedRoutedExpertWeights::Mlx(weights) => {
                self.kernels.gate_up_mlx(
                    stream,
                    input,
                    u32s(&weights.gate_blocks)?,
                    u8s(&weights.gate_scales)?,
                    bf16s(&weights.gate_bias)?,
                    u32s(&weights.up_blocks)?,
                    u8s(&weights.up_scales)?,
                    bf16s(&weights.up_bias)?,
                    selected,
                    activated,
                )?;
                match self.execution {
                    ClampedMoeExecution::RouteParallel => self.kernels.down_routes_mlx(
                        stream,
                        activated,
                        u32s(&weights.down_blocks)?,
                        u8s(&weights.down_scales)?,
                        bf16s(&weights.down_bias)?,
                        selected,
                        routing,
                        partial,
                        output,
                    ),
                    ClampedMoeExecution::FusedReduce => self.kernels.down_mlx(
                        stream,
                        activated,
                        u32s(&weights.down_blocks)?,
                        u8s(&weights.down_scales)?,
                        bf16s(&weights.down_bias)?,
                        selected,
                        routing,
                        output,
                    ),
                }
            },
            ClampedRoutedExpertWeights::Dense(_) => {
                Err(Error::InvalidExecutionPlan("dense experts cannot use clamped MXFP4 execution"))
            },
        }
    }
}

fn bf16s(tensor: &CudaTensor) -> Result<&DeviceBuffer<bf16>> {
    tensor.as_bf16().ok_or_else(|| Error::DTypeMismatch {
        name: tensor.name().into(),
        expected: "BF16",
    })
}

fn u8s(tensor: &CudaTensor) -> Result<&DeviceBuffer<u8>> {
    tensor.as_u8().ok_or_else(|| Error::DTypeMismatch {
        name: tensor.name().into(),
        expected: "U8",
    })
}

fn u32s(tensor: &CudaTensor) -> Result<&DeviceBuffer<u32>> {
    tensor.as_u32().ok_or_else(|| Error::DTypeMismatch {
        name: tensor.name().into(),
        expected: "U32",
    })
}