libmir-cuda 0.2.0

CUDA inference backend for libmir
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83

use models::{
    layout::{EncoderConfig, EncoderRopeScaling, NormKind},
    weights::{TensorCatalog, TensorInfo},
};

use super::CudaSequenceScoringModel;
use crate::{CudaBackend, Error, Result};

impl CudaSequenceScoringModel {
    pub fn load(
        backend: &CudaBackend,
        config: &EncoderConfig,
        catalog: &TensorCatalog,
    ) -> Result<Self> {
        validate(config)?;
        let mut upload = backend.begin_tensor_upload();
        for name in required_names(config) {
            upload.enqueue(required(catalog, &name)?)?;
        }
        Ok(Self {
            backend: backend.clone(),
            config: config.clone(),
            tensors: upload.finish()?,
        })
    }
}

fn validate(config: &EncoderConfig) -> Result<()> {
    if !config.packed_qkv
        || config.norm != NormKind::LayerNorm
        || config.hidden_activation != "gelu"
        || config.num_labels != 1
        || !matches!(config.rope_scaling, Some(EncoderRopeScaling::Ntk { mixed_b: None, .. }))
    {
        return Err(Error::UnsupportedDecoderLayer(
            "CUDA sequence scoring requires packed QKV, LayerNorm, GELU, and fixed NTK RoPE".into(),
        ));
    }
    Ok(())
}

fn required_names(config: &EncoderConfig) -> Vec<String> {
    let mut names: Vec<String> = [
        "new.embeddings.word_embeddings.weight",
        "new.embeddings.token_type_embeddings.weight",
        "new.embeddings.LayerNorm.weight",
        "new.embeddings.LayerNorm.bias",
        "new.pooler.dense.weight",
        "new.pooler.dense.bias",
        "classifier.weight",
        "classifier.bias",
    ]
    .into_iter()
    .map(Into::into)
    .collect();
    for layer in 0..config.num_hidden_layers {
        let prefix = format!("new.encoder.layer.{layer}");
        for suffix in [
            "attention.qkv_proj.weight",
            "attention.qkv_proj.bias",
            "attention.o_proj.weight",
            "attention.o_proj.bias",
            "attn_ln.weight",
            "attn_ln.bias",
            "mlp.up_gate_proj.weight",
            "mlp.down_proj.weight",
            "mlp.down_proj.bias",
            "mlp_ln.weight",
            "mlp_ln.bias",
        ] {
            names.push(format!("{prefix}.{suffix}"));
        }
    }
    names
}

fn required<'a>(catalog: &'a TensorCatalog, name: &str) -> Result<&'a TensorInfo> {
    catalog
        .tensors
        .iter()
        .find(|tensor| tensor.name == name)
        .ok_or_else(|| Error::MissingTensor(name.into()))
}