libmir-cuda 0.3.0

CUDA inference backend for libmir
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73

use mircuda::{
    CompileOptions, Compiler, DeviceBuffer, LaunchConfig, Stream, TypedKernel, bf16, cuda_export,
    cuda_kernel_file, f16,
};

use crate::{Error, Result};

cuda_export!(
    F16ToBf16Kernel = "libmir_cuda_dense_f16_to_bf16"(
        input: &DeviceBuffer<f16>, output: &mut DeviceBuffer<bf16>, elements: u32,
    )
);
cuda_export!(
    F32ToBf16Kernel = "libmir_cuda_dense_f32_to_bf16"(
        input: &DeviceBuffer<f32>, output: &mut DeviceBuffer<bf16>, elements: u32,
    )
);

#[derive(Debug)]
pub struct DenseCast {
    f16_to_bf16: TypedKernel<F16ToBf16Kernel>,
    f32_to_bf16: TypedKernel<F32ToBf16Kernel>,
}

impl DenseCast {
    pub fn compile(compiler: &Compiler) -> Result<Self> {
        let source = cuda_kernel_file!("../../kernels/dense_cast.cu");
        let module = compiler.compile(source, &CompileOptions::default())?;
        Ok(Self {
            f16_to_bf16: module.kernel()?,
            f32_to_bf16: module.kernel()?,
        })
    }

    pub fn f16_to_bf16(
        &self,
        stream: &Stream,
        input: &DeviceBuffer<f16>,
        output: &mut DeviceBuffer<bf16>,
    ) -> Result<()> {
        validate(input.len(), output.len())?;
        Ok(self.f16_to_bf16.launch(
            stream,
            launch(input.len())?,
            (input, output, u32::try_from(input.len())?),
        )?)
    }

    pub fn f32_to_bf16(
        &self,
        stream: &Stream,
        input: &DeviceBuffer<f32>,
        output: &mut DeviceBuffer<bf16>,
    ) -> Result<()> {
        validate(input.len(), output.len())?;
        Ok(self.f32_to_bf16.launch(
            stream,
            launch(input.len())?,
            (input, output, u32::try_from(input.len())?),
        )?)
    }
}

fn validate(input: usize, output: usize) -> Result<()> {
    if input == 0 || input != output {
        return Err(Error::InvalidTensorConversion { input, output });
    }
    Ok(())
}

fn launch(elements: usize) -> Result<LaunchConfig> {
    Ok(LaunchConfig::for_elements(elements, 256)?)
}