libmir-metal 0.1.0

Metal inference backend for libmir
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77

mod array;
mod attention;
mod compiled;
mod decode_graph;
mod decoder;
mod decoder_cache;
pub mod dense_swiglu;
mod embedding;
mod error;
pub(crate) mod expert_fusion;
mod fused_attention;
mod fused_expert_gate_up;
mod fused_gate_up;
mod fused_key_value;
mod gated_delta;
mod gated_delta_layer;
mod gated_full_attention;
mod graph;
pub mod hybrid_linear_moe;
pub mod hybrid_moe;
mod kernels;
mod kv;
mod linear;
mod memory;
mod metadata;
mod model;
mod moe;
mod norm;
mod quantized;
mod sampling;
mod scalar;
mod shared_expert_moe;
mod snapshot;
mod stream;
mod tensors;
#[cfg(test)]
mod tests;

pub use array::Array;
pub use attention::{PagedAttention, RopeOptions};
pub(crate) use decoder::DecoderModel;
pub use decoder_cache::DecoderCache;
pub use embedding::QuantizedEmbedding;
pub use error::{Error, Result};
pub(crate) use expert_fusion::{ExpertFusion, ExpertFusionDecision, configure_expert_fusion};
pub(crate) use fused_attention::FusedAttention;
pub(crate) use fused_expert_gate_up::FusedExpertGateUp;
pub(crate) use fused_gate_up::FusedGateUp;
pub(crate) use fused_key_value::FusedKeyValue;
pub use gated_delta::{GatedDeltaInputs, GatedDeltaState};
pub use gated_delta_layer::{GatedDeltaLayer, GatedDeltaLayerConfig};
pub use gated_full_attention::{GatedFullAttention, GatedFullAttentionConfig};
pub use kv::{KvCache, KvContext, PagedKvContext};
pub(crate) use kv::{
    NATIVE_PAGED_ATTENTION_MIN_CONTEXT, PagedContextMode, native_paged_attention_mode,
    paged_attention_enabled, paged_attention_min_context,
};
pub use linear::QuantizedLinear;
pub(crate) use memory::{MemoryStats, configure_recommended_wired_limit, memory_stats};
pub use metadata::Dtype;
pub use model::ModelTensors;
pub use moe::{RouterOutput, SortedExpertInputs};
pub(crate) use norm::NormWeight;
pub use quantized::QuantizedArrays;
pub use sampling::TopK;
pub(crate) use sampling::{DeviceSampling, sample, sample_u32};
pub use shared_expert_moe::{SharedExpertMoe, SharedExpertMoeConfig};
pub use stream::Stream;
pub use tensors::TensorFile;

pub fn version() -> Result<String> {
    Ok(mirtal::version()?)
}

pub fn clear_memory_cache() -> Result<()> {
    Ok(mirtal::memory::clear_cache()?)
}