moe-llm-core 1.3.6

Part of the MoE-13 Ternary Intelligence Stack
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40

#[derive(Clone, Debug)]
pub struct TransformerConfig {
    pub vocab_size: usize,
    pub hidden_size: usize,
    pub num_layers: usize,
    pub num_heads: usize,
    pub max_seq_len: usize,
    pub threshold: f32,
    pub num_experts: usize,
}

impl Default for TransformerConfig {
    fn default() -> Self {
        Self {
            vocab_size: 8000,
            hidden_size: 64,
            num_layers: 2,
            num_heads: 4,
            max_seq_len: 64,
            threshold: 0.02,
            num_experts: 8,
        }
    }
}

impl TransformerConfig {
    /// Per-layer ternary threshold: early layers stay dense (small threshold = more weights active),
    /// deeper layers grow sparse (larger threshold = more weights zeroed → faster inference).
    /// Range: 0.01 (layer 0) → up to 0.05 (deepest layer).
    pub fn layer_threshold(&self, layer_idx: usize) -> f32 {
        let base = 0.01_f32;
        let top  = self.threshold.max(0.03);
        let step = if self.num_layers > 1 {
            (top - base) / (self.num_layers - 1) as f32
        } else {
            0.0
        };
        (base + step * layer_idx as f32).min(0.05)
    }
}