libmir-cuda 0.3.0

CUDA inference backend for libmir
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28

use mircuda::{DeviceBuffer, bf16, cuda_export};

cuda_export!(
    pub(super) AffineGemvInt4Kernel = "libmir_cuda_affine_gemv_bf16_int4"(
        input: &DeviceBuffer<bf16>, weight: &DeviceBuffer<u32>,
        scales: &DeviceBuffer<bf16>, biases: &DeviceBuffer<bf16>,
        output: &mut DeviceBuffer<bf16>, input_features: u32,
        output_features: u32, group_size: u32, matrix_index: u32,
    )
);

cuda_export!(
    pub(super) AffineGemvInt8Kernel = "libmir_cuda_affine_gemv_bf16_int8"(
        input: &DeviceBuffer<bf16>, weight: &DeviceBuffer<u32>,
        scales: &DeviceBuffer<bf16>, biases: &DeviceBuffer<bf16>,
        output: &mut DeviceBuffer<bf16>, input_features: u32,
        output_features: u32, group_size: u32, matrix_index: u32,
    )
);

cuda_export!(
    pub(super) AffineGemvFallbackKernel = "libmir_cuda_affine_gemv_bf16_fallback"(
        input: &DeviceBuffer<bf16>, weight: &DeviceBuffer<u32>,
        scales: &DeviceBuffer<bf16>, biases: &DeviceBuffer<bf16>,
        output: &mut DeviceBuffer<bf16>, input_features: u32,
        output_features: u32, group_size: u32, matrix_index: u32,
    )
);