libmir-cuda 0.2.0

CUDA inference backend for libmir
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48

use mircuda::{DeviceBuffer, bf16};

use super::{AffineGatedDeltaLayerConfig, checked};
use crate::{CudaBackend, Result};

#[derive(Debug)]
pub(super) struct GatedDeltaScratch {
    pub(super) mixed: DeviceBuffer<bf16>,
    pub(super) convolved: DeviceBuffer<bf16>,
    pub(super) query: DeviceBuffer<bf16>,
    pub(super) key: DeviceBuffer<bf16>,
    pub(super) value: DeviceBuffer<bf16>,
    pub(super) normalized_query: DeviceBuffer<bf16>,
    pub(super) normalized_key: DeviceBuffer<bf16>,
    pub(super) gate: DeviceBuffer<bf16>,
    pub(super) alpha: DeviceBuffer<bf16>,
    pub(super) beta: DeviceBuffer<bf16>,
    pub(super) recurrent: DeviceBuffer<bf16>,
    pub(super) gated: DeviceBuffer<bf16>,
}

impl GatedDeltaScratch {
    pub(super) fn new(
        backend: &CudaBackend,
        config: AffineGatedDeltaLayerConfig,
        tokens: usize,
    ) -> Result<Self> {
        let mixed = checked(tokens, config.mixed_width()?)?;
        let key = checked(tokens, config.key_width()?)?;
        let value = checked(tokens, config.value_width()?)?;
        let heads = checked(tokens, config.value_heads)?;
        let allocate = |elements| backend.inner.pool.allocate(&backend.inner.stream, elements);
        Ok(Self {
            mixed: allocate(mixed)?,
            convolved: allocate(mixed)?,
            query: allocate(key)?,
            key: allocate(key)?,
            value: allocate(value)?,
            normalized_query: allocate(key)?,
            normalized_key: allocate(key)?,
            gate: allocate(value)?,
            alpha: allocate(heads)?,
            beta: allocate(heads)?,
            recurrent: allocate(value)?,
            gated: allocate(value)?,
        })
    }
}