libmir-cuda 0.2.0

CUDA inference backend for libmir
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52

use std::path::PathBuf;

use models::{
    layout::DecoderConfig,
    semantic::SemanticModelSpec,
    weights::{TensorCatalog, TensorInfo},
};
use serde_json::json;

use super::*;

#[test]
fn lowers_dense_qk_normalization_from_semantics() -> Result<()> {
    let decoder = DecoderConfig::from_value(&json!({
        "hidden_size": 8,
        "intermediate_size": 16,
        "num_hidden_layers": 1,
        "num_attention_heads": 2,
        "num_key_value_heads": 1,
        "head_dim": 4,
        "vocab_size": 32,
        "hidden_act": "silu",
        "model_type": "misleading"
    }))?;
    let catalog = TensorCatalog {
        tensors: [
            "model.layers.0.input_layernorm.weight",
            "model.layers.0.self_attn.q_norm.weight",
            "model.layers.0.self_attn.k_norm.weight",
        ]
        .into_iter()
        .map(tensor)
        .collect(),
    };
    let spec = SemanticModelSpec::discover(&decoder, &catalog)?;
    let plan = CudaDecoderPlan::lower(&spec);

    assert!(plan.all_dense());
    assert_eq!(plan.graph_normalization()?, KernelNormalization::QUERY_KEY);
    Ok(())
}

fn tensor(name: &str) -> TensorInfo {
    TensorInfo {
        name: name.into(),
        file: PathBuf::new(),
        dtype: "BF16".into(),
        shape: vec![8],
        data_start: 0,
        data_offsets: [0, 0],
    }
}