libmir-metal 0.2.0

Metal inference backend for libmir
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115

use models::{
    layout::DecoderConfig,
    semantic::{
        ActivationSpec, AttentionOutputSpec, FeedForwardSpec, LinearAttentionSpec, MixerSpec,
        NormalizationKind, SemanticModelSpec,
    },
    weights::{TensorCatalog, TensorInfo},
};
use serde_json::json;

use super::*;

#[test]
fn materializes_dense_layers_without_model_identity() -> Result<()> {
    let spec = dense_spec()?;
    let lowering = plan(&spec)?;

    assert_eq!(
        lowering.layers(),
        &[LayerLowering {
            index: 0,
            input_norm: NormalizationLowering::Rms,
            post_attention_norm: NormalizationLowering::Rms,
            mixer: MixerLowering::Softmax { sinks: false, window: None },
            feed_forward: FeedForwardLowering::Dense,
        }]
    );
    assert_eq!(lowering.runtime(), DecoderRuntime::Dense);
    Ok(())
}

#[test]
fn rejects_an_unavailable_normalization_independently() -> Result<()> {
    let mut spec = dense_spec()?;
    spec.decoder.layers[0].input_norm.kind = NormalizationKind::Layer;

    let error = lower_layer(&spec.decoder.layers[0])
        .err()
        .ok_or_else(|| Error::InvalidModel("layer normalization was admitted".into()))?;

    assert!(error.to_string().contains("input layer normalization"));
    Ok(())
}

#[test]
fn does_not_admit_linear_attention_through_the_dense_runtime() -> Result<()> {
    let mut spec = dense_spec()?;
    spec.decoder.layers[0].mixer = MixerSpec::LinearAttention(LinearAttentionSpec {
        convolution_kernel_size: 4,
        key_heads: 2,
        value_heads: 2,
        key_head_dim: 4,
        value_head_dim: 4,
        output: AttentionOutputSpec::Direct,
    });

    assert_eq!(lower_layer(&spec.decoder.layers[0])?.mixer, MixerLowering::Linear);
    assert!(plan(&spec).is_err());
    Ok(())
}

#[test]
fn preserves_windows_and_rejects_them_from_the_dense_runtime() -> Result<()> {
    let mut spec = dense_spec()?;
    if let MixerSpec::SoftmaxAttention(attention) = &mut spec.decoder.layers[0].mixer {
        attention.window = Some(128);
    }

    assert_eq!(
        lower_layer(&spec.decoder.layers[0])?.mixer,
        MixerLowering::Softmax { sinks: false, window: Some(128) }
    );
    assert!(plan(&spec).is_err());
    Ok(())
}

#[test]
fn rejects_an_activation_the_dense_operator_does_not_implement() -> Result<()> {
    let mut spec = dense_spec()?;
    if let FeedForwardSpec::Dense { activation, .. } = &mut spec.decoder.layers[0].feed_forward {
        *activation = ActivationSpec::GeluTanh;
    }

    let error = lower_layer(&spec.decoder.layers[0])
        .err()
        .ok_or_else(|| Error::InvalidModel("dense GELU was admitted".into()))?;

    assert!(error.to_string().contains("feed-forward activation composition"));
    Ok(())
}

fn dense_spec() -> Result<SemanticModelSpec> {
    let decoder = DecoderConfig::from_value(&json!({
        "hidden_size": 8,
        "intermediate_size": 16,
        "num_hidden_layers": 1,
        "num_attention_heads": 2,
        "num_key_value_heads": 1,
        "head_dim": 4,
        "vocab_size": 32,
        "hidden_act": "silu",
        "model_type": "misleading"
    }))?;
    let catalog = TensorCatalog {
        tensors: vec![TensorInfo {
            name: "model.layers.0.input_layernorm.weight".into(),
            file: std::path::PathBuf::new(),
            dtype: "BF16".into(),
            shape: vec![8],
            data_start: 0,
            data_offsets: [0, 0],
        }],
    };
    Ok(SemanticModelSpec::discover(&decoder, &catalog)?)
}