libmir-cuda 0.2.0

CUDA inference backend for libmir
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91

use std::{fs, path::PathBuf};

use mircuda::bf16;
use models::weights::TensorInfo;

use super::super::super::AffineGatedFullAttentionConfig;
use crate::{CudaBackend, CudaTensorSet, Result};

pub(super) const PREFIX: &str = "language_model.model.layers.0.self_attn";

pub(super) struct AttentionFixture {
    path: PathBuf,
    bytes: Vec<u8>,
    infos: Vec<TensorInfo>,
}

impl AttentionFixture {
    pub(super) fn new(config: AffineGatedFullAttentionConfig) -> Result<Self> {
        let path = std::env::temp_dir()
            .join(format!("libmir-cuda-gated-attention-{}.bin", std::process::id()));
        let mut fixture = Self {
            path,
            bytes: Vec::new(),
            infos: Vec::new(),
        };
        let query = config.query_width()?;
        let key_value = config.key_value_width()?;
        fixture.projection("q_proj", config.hidden_size, 2 * query, config)?;
        fixture.projection("k_proj", config.hidden_size, key_value, config)?;
        fixture.projection("v_proj", config.hidden_size, key_value, config)?;
        fixture.projection("o_proj", query, config.hidden_size, config)?;
        fixture.bf16("q_norm.weight", vec![config.head_dim], vec![1.0; config.head_dim])?;
        fixture.bf16("k_norm.weight", vec![config.head_dim], vec![1.0; config.head_dim])?;
        fs::write(&fixture.path, &fixture.bytes)?;
        Ok(fixture)
    }

    pub(super) fn upload(&self, backend: &CudaBackend) -> Result<CudaTensorSet> {
        let mut upload = backend.begin_tensor_upload();
        for info in &self.infos {
            upload.enqueue(info)?;
        }
        upload.finish()
    }

    fn projection(
        &mut self,
        name: &str,
        input: usize,
        output: usize,
        config: AffineGatedFullAttentionConfig,
    ) -> Result<()> {
        let packed = input / (32 / config.bits);
        let groups = input / config.group_size;
        self.u32(&format!("{name}.weight"), vec![output, packed], output * packed)?;
        self.bf16(&format!("{name}.scales"), vec![output, groups], vec![0.0; output * groups])?;
        self.bf16(&format!("{name}.biases"), vec![output, groups], vec![0.0; output * groups])
    }

    fn u32(&mut self, name: &str, shape: Vec<usize>, elements: usize) -> Result<()> {
        let start = u64::try_from(self.bytes.len())?;
        self.bytes.resize(self.bytes.len() + elements * 4, 0);
        self.info(name, "U32", shape, start)
    }

    fn bf16(&mut self, name: &str, shape: Vec<usize>, values: Vec<f32>) -> Result<()> {
        let start = u64::try_from(self.bytes.len())?;
        for value in values {
            self.bytes.extend_from_slice(&bf16::from_f32(value).to_bits().to_le_bytes());
        }
        self.info(name, "BF16", shape, start)
    }

    fn info(&mut self, name: &str, dtype: &str, shape: Vec<usize>, start: u64) -> Result<()> {
        self.infos.push(TensorInfo {
            name: format!("{PREFIX}.{name}"),
            file: self.path.clone(),
            dtype: dtype.into(),
            shape,
            data_start: 0,
            data_offsets: [start, u64::try_from(self.bytes.len())?],
        });
        Ok(())
    }
}

impl Drop for AttentionFixture {
    fn drop(&mut self) {
        let _removed = fs::remove_file(&self.path);
    }
}