libmir-metal 0.2.0

Metal inference backend for libmir
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87

mod array;
mod attention;
mod binding;
pub mod clamped_routed;
mod compiled;
mod decode_graph;
mod decoder;
pub mod dense;
mod embedding;
mod error;
pub(crate) mod expert_fusion;
mod fused_attention;
mod fused_expert_gate_up;
mod fused_gate_up;
mod fused_key_value;
mod fusion_planner;
mod gated_delta;
mod gated_full_attention;
mod graph;
pub mod hybrid_linear_moe;
pub mod hybrid_moe;
mod kernels;
mod kv;
mod layer_norm;
mod linear;
pub mod lowering;
mod memory;
mod metadata;
mod model;
mod moe;
mod norm;
mod quantized;
mod sampling;
mod scalar;
mod sequence_scoring;
mod shared_expert_moe;
mod snapshot;
mod stream;
mod tensors;
#[cfg(test)]
mod tests;
mod text_embedding;
pub mod vision;

pub use array::Array;
pub use attention::{ImageTokenSpan, PagedAttention, RopeOptions, prefix_attention_mask};
pub use decoder::DecoderCache;
pub(crate) use decoder::DecoderModel;
pub use dense::{DenseEmbedding, DenseLinear};
pub use embedding::QuantizedEmbedding;
pub use error::{Error, Result};
pub(crate) use expert_fusion::{ExpertFusion, ExpertFusionDecision, configure_expert_fusion};
pub(crate) use fused_attention::FusedAttention;
pub(crate) use fused_expert_gate_up::FusedExpertGateUp;
pub(crate) use fused_gate_up::FusedGateUp;
pub(crate) use fused_key_value::FusedKeyValue;
pub use gated_delta::{GatedDeltaInputs, GatedDeltaLayer, GatedDeltaLayerConfig, GatedDeltaState};
pub use gated_full_attention::{GatedFullAttention, GatedFullAttentionConfig};
pub use kv::{KvCache, KvContext, PagedKvContext};
pub(crate) use kv::{
    KvPageFormat, NATIVE_PAGED_ATTENTION_MIN_CONTEXT, PagedContextMode,
    native_paged_attention_mode, paged_attention_enabled, paged_attention_min_context,
};
pub use layer_norm::LayerNorm;
pub use linear::QuantizedLinear;
pub(crate) use memory::{MemoryStats, configure_recommended_wired_limit, memory_stats};
pub use metadata::Dtype;
pub use model::ModelTensors;
pub use moe::{RouterOutput, SortedExpertInputs};
pub(crate) use norm::NormWeight;
pub use quantized::QuantizedArrays;
pub use sampling::TopK;
pub(crate) use sampling::{DeviceSampling, sample, sample_u32};
pub use sequence_scoring::SequenceScoringModel;
pub use shared_expert_moe::{SharedExpertMoe, SharedExpertMoeConfig};
pub use stream::Stream;
pub use tensors::TensorFile;
pub use text_embedding::TextEmbeddingModel;
pub use vision::{pooled::PooledVisionTower, spatial_merge::SpatialMergeVisionTower};

pub fn version() -> Result<String> {
    Ok(mirtal::version()?)
}

pub fn clear_memory_cache() -> Result<()> {
    Ok(mirtal::memory::clear_cache()?)
}