libmir-cuda 0.2.0

CUDA inference backend for libmir
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53

use mircuda::{DeviceBuffer, bf16};

use crate::{CudaBackend, Error, Result};

#[derive(Debug)]
pub(super) struct PrefillBlockScratch {
    pub(super) attention: DeviceBuffer<bf16>,
    pub(super) attention_norm: DeviceBuffer<bf16>,
    pub(super) hidden: DeviceBuffer<bf16>,
    pub(super) normalized: DeviceBuffer<bf16>,
    pub(super) dense_gate_up: DeviceBuffer<bf16>,
    pub(super) dense_activated: DeviceBuffer<bf16>,
    pub(super) dense: DeviceBuffer<bf16>,
    pub(super) expert: DeviceBuffer<bf16>,
    pub(super) expert_norm: DeviceBuffer<bf16>,
    pub(super) feed_forward: DeviceBuffer<bf16>,
    pub(super) feed_forward_norm: DeviceBuffer<bf16>,
    pub(super) residual: DeviceBuffer<bf16>,
}

impl PrefillBlockScratch {
    pub(super) fn new(
        backend: &CudaBackend,
        tokens: usize,
        hidden: usize,
        dense: usize,
    ) -> Result<Self> {
        let hidden = tokens
            .checked_mul(hidden)
            .ok_or(Error::InvalidDecoderKernel("prefill hidden scratch size overflow"))?;
        let dense = tokens
            .checked_mul(dense)
            .ok_or(Error::InvalidDecoderKernel("prefill dense scratch size overflow"))?;
        let dense_pair = dense
            .checked_mul(2)
            .ok_or(Error::InvalidDecoderKernel("prefill dense pair size overflow"))?;
        let allocate = |elements| backend.inner.pool.allocate(&backend.inner.stream, elements);
        Ok(Self {
            attention: allocate(hidden)?,
            attention_norm: allocate(hidden)?,
            hidden: allocate(hidden)?,
            normalized: allocate(hidden)?,
            dense_gate_up: allocate(dense_pair)?,
            dense_activated: allocate(dense)?,
            dense: allocate(hidden)?,
            expert: allocate(hidden)?,
            expert_norm: allocate(hidden)?,
            feed_forward: allocate(hidden)?,
            feed_forward_norm: allocate(hidden)?,
            residual: allocate(hidden)?,
        })
    }
}