libmir-cuda 0.1.0

CUDA inference backend for libmir
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123

mod quantize;

use mircuda::{
    CompileOptions, Compiler, DeviceBuffer, LaunchConfig, Stream, TypedKernel, bf16, cuda_export,
    cuda_kernel_file,
};

use super::geometry::{narrow, product, require};
use crate::{Error, Result};

cuda_export!(PrepareBucketsKernel = "libmir_cuda_nvfp4_prepare_buckets"(
    selected: &DeviceBuffer<u32>, counts: &mut DeviceBuffer<u32>,
    offsets: &mut DeviceBuffer<u32>, order: &mut DeviceBuffer<u32>,
    positions: &mut DeviceBuffer<u32>, indices: &mut DeviceBuffer<u32>,
    assignments: u32, experts: u32,
));
cuda_export!(QuantizeBucketsKernel = "libmir_cuda_nvfp4_quantize_buckets_bf16"(
    input: &DeviceBuffer<bf16>, selected: &DeviceBuffer<u32>,
    order: &DeviceBuffer<u32>, offsets: &DeviceBuffer<u32>,
    global_scales: &DeviceBuffer<f32>, packed: &mut DeviceBuffer<u8>,
    scales: &mut DeviceBuffer<u8>, assignments: u32, selected_count: u32,
    input_rows: u32, columns: u32, scale_stride: u32, ranked: u32,
));
cuda_export!(QuantizeBucketPairKernel = "libmir_cuda_nvfp4_quantize_bucket_pair_bf16"(
    input: &DeviceBuffer<bf16>, selected: &DeviceBuffer<u32>,
    order: &DeviceBuffer<u32>, offsets: &DeviceBuffer<u32>,
    left_globals: &DeviceBuffer<f32>, right_globals: &DeviceBuffer<f32>,
    left_packed: &mut DeviceBuffer<u8>, right_packed: &mut DeviceBuffer<u8>,
    left_scales: &mut DeviceBuffer<u8>, right_scales: &mut DeviceBuffer<u8>,
    assignments: u32, selected_count: u32, input_rows: u32,
    columns: u32, scale_stride: u32,
));

#[derive(Clone, Debug)]
pub struct NvFp4BucketPreparation {
    prepare: TypedKernel<PrepareBucketsKernel>,
    quantize: TypedKernel<QuantizeBucketsKernel>,
    quantize_pair: TypedKernel<QuantizeBucketPairKernel>,
}

impl NvFp4BucketPreparation {
    pub fn compile(compiler: &Compiler) -> Result<Self> {
        let source = cuda_kernel_file!("../../../kernels/nvfp4_buckets.cu");
        let options = CompileOptions { fast_math: false, ..Default::default() };
        let module = compiler.compile(source, &options)?;
        Ok(Self {
            prepare: module.kernel()?,
            quantize: module.kernel()?,
            quantize_pair: module.kernel()?,
        })
    }

    #[allow(clippy::too_many_arguments)]
    pub fn prepare(
        &self,
        stream: &Stream,
        selected: &DeviceBuffer<u32>,
        counts: &mut DeviceBuffer<u32>,
        offsets: &mut DeviceBuffer<u32>,
        order: &mut DeviceBuffer<u32>,
        positions: &mut DeviceBuffer<u32>,
        indices: &mut DeviceBuffer<u32>,
        geometry: BucketGeometry,
    ) -> Result<()> {
        geometry.validate(selected, counts, offsets, order, positions, indices)?;
        Ok(self.prepare.launch(
            stream,
            LaunchConfig {
                grid: (1, 1, 1),
                block: (256, 1, 1),
                shared_memory_bytes: narrow(product(geometry.experts, 2 * size_of::<u32>())?)?,
            },
            (
                selected,
                counts,
                offsets,
                order,
                positions,
                indices,
                narrow(geometry.assignments)?,
                narrow(geometry.experts)?,
            ),
        )?)
    }
}

#[derive(Clone, Copy, Debug)]
pub struct BucketGeometry {
    pub assignments: usize,
    pub experts: usize,
}

impl BucketGeometry {
    fn validate(
        self,
        selected: &DeviceBuffer<u32>,
        counts: &DeviceBuffer<u32>,
        offsets: &DeviceBuffer<u32>,
        order: &DeviceBuffer<u32>,
        positions: &DeviceBuffer<u32>,
        indices: &DeviceBuffer<u32>,
    ) -> Result<()> {
        if self.assignments == 0 || self.experts == 0 {
            return Err(Error::InvalidNvFp4("invalid bucket geometry"));
        }
        require("bucket selections", self.assignments, selected.len())?;
        require("bucket counts", self.experts, counts.len())?;
        require("bucket offsets", self.experts, offsets.len())?;
        require("bucket order", self.assignments, order.len())?;
        require("bucket positions", self.assignments, positions.len())?;
        require("bucket indices", self.experts, indices.len())
    }
}

#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub struct BucketQuantize {
    pub assignments: usize,
    pub experts: usize,
    pub selected: usize,
    pub input_rows: usize,
    pub columns: usize,
    pub ranked: bool,
}