libmir-cuda 0.2.0

CUDA inference backend for libmir
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57

use mircuda::{
    CompileOptions, Compiler, DeviceBuffer, LaunchConfig, Stream, TypedKernel, bf16, cuda_export,
    cuda_kernel_file,
};

use super::super::geometry::{narrow, product, require};
use crate::Result;

cuda_export!(
    PatchLayoutKernel = "libmir_cuda_vision_patch_cthw_to_thwc_bf16"(
        input: &DeviceBuffer<f32>, output: &mut DeviceBuffer<bf16>, elements: u32,
        channels: u32, temporal: u32, patch_area: u32,
    )
);

#[derive(Clone, Debug)]
pub struct VisionPatchLayout {
    kernel: TypedKernel<PatchLayoutKernel>,
}

impl VisionPatchLayout {
    pub fn compile(compiler: &Compiler) -> Result<Self> {
        let source = cuda_kernel_file!("../../../kernels/vision_patch_layout.cu");
        let module = compiler.compile(source, &CompileOptions::default())?;
        Ok(Self { kernel: module.kernel()? })
    }

    pub fn cthw_to_thwc(
        &self,
        stream: &Stream,
        input: &DeviceBuffer<f32>,
        output: &mut DeviceBuffer<bf16>,
        geometry: [usize; 4],
    ) -> Result<()> {
        let [tokens, channels, temporal, patch_area] = geometry;
        let elements = product(product(product(tokens, channels)?, temporal)?, patch_area)?;
        require("vision patch layout input", elements, input.len())?;
        require("vision patch layout output", elements, output.len())?;
        let threads = 256_usize;
        Ok(self.kernel.launch(
            stream,
            LaunchConfig {
                grid: (narrow(elements.div_ceil(threads))?, 1, 1),
                block: (narrow(threads)?, 1, 1),
                shared_memory_bytes: 0,
            },
            (
                input,
                output,
                narrow(elements)?,
                narrow(channels)?,
                narrow(temporal)?,
                narrow(patch_area)?,
            ),
        )?)
    }
}