libmir-cuda 0.2.0

CUDA inference backend for libmir
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100

use mircuda::{
    CompileOptions, Compiler, DeviceBuffer, LaunchConfig, Stream, TypedKernel, bf16, cuda_export,
    cuda_kernel_file,
};

use super::super::geometry::{narrow, product, require};
use crate::{Error, Result};

cuda_export!(
    ConvolutionKernel = "libmir_cuda_gated_delta_convolution_bf16"(
        input: &DeviceBuffer<bf16>, weight: &DeviceBuffer<bf16>,
        history: &DeviceBuffer<bf16>, output: &mut DeviceBuffer<bf16>,
        tokens: u32, channels: u32, kernel_size: u32,
    )
);
cuda_export!(
    HistoryKernel = "libmir_cuda_gated_delta_history_bf16"(
        input: &DeviceBuffer<bf16>, history: &DeviceBuffer<bf16>,
        next_history: &mut DeviceBuffer<bf16>, tokens: u32,
        channels: u32, kernel_size: u32,
    )
);

#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub struct GatedDeltaConvolutionSpec {
    pub tokens: usize,
    pub channels: usize,
    pub kernel_size: usize,
}

#[derive(Clone, Debug)]
pub struct GatedDeltaConvolution {
    convolution: TypedKernel<ConvolutionKernel>,
    history: TypedKernel<HistoryKernel>,
    spec: GatedDeltaConvolutionSpec,
}

impl GatedDeltaConvolution {
    pub fn compile(compiler: &Compiler, spec: GatedDeltaConvolutionSpec) -> Result<Self> {
        if spec.tokens == 0 || spec.channels == 0 || spec.kernel_size < 2 {
            return Err(Error::InvalidDecoderKernel("invalid Gated Delta convolution geometry"));
        }
        let source = cuda_kernel_file!("../../../kernels/gated_delta_bf16.cu");
        let module = compiler.compile(source, &CompileOptions::default())?;
        Ok(Self {
            convolution: module.kernel()?,
            history: module.kernel()?,
            spec,
        })
    }

    pub fn execute(
        &self,
        stream: &Stream,
        input: &DeviceBuffer<bf16>,
        weight: &DeviceBuffer<bf16>,
        history: &DeviceBuffer<bf16>,
        next_history: &mut DeviceBuffer<bf16>,
        output: &mut DeviceBuffer<bf16>,
    ) -> Result<()> {
        let values = product(self.spec.tokens, self.spec.channels)?;
        let history_values = self.history_elements()?;
        require("Gated Delta convolution input", values, input.len())?;
        require(
            "Gated Delta convolution weight",
            product(self.spec.channels, self.spec.kernel_size)?,
            weight.len(),
        )?;
        require("Gated Delta convolution history", history_values, history.len())?;
        require("Gated Delta next convolution history", history_values, next_history.len())?;
        require("Gated Delta convolution output", values, output.len())?;
        let arguments = (
            narrow(self.spec.tokens)?,
            narrow(self.spec.channels)?,
            narrow(self.spec.kernel_size)?,
        );
        self.convolution.launch(
            stream,
            launch(values)?,
            (input, weight, history, output, arguments.0, arguments.1, arguments.2),
        )?;
        Ok(self.history.launch(
            stream,
            launch(history_values)?,
            (input, history, next_history, arguments.0, arguments.1, arguments.2),
        )?)
    }

    pub fn history_elements(&self) -> Result<usize> {
        product(self.spec.kernel_size - 1, self.spec.channels)
    }
}

fn launch(elements: usize) -> Result<LaunchConfig> {
    Ok(LaunchConfig {
        grid: (narrow(elements.div_ceil(256))?, 1, 1),
        block: (256, 1, 1),
        shared_memory_bytes: 0,
    })
}