moe-llm-core 1.3.5

Part of the MoE-13 Ternary Intelligence Stack
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19

pub struct TransformerConfig {
    pub vocab_size: usize,
    pub hidden_size: usize,
    pub num_layers: usize,
    pub num_heads: usize,
    pub max_seq_len: usize,
}

impl Default for TransformerConfig {
    fn default() -> Self {
        Self {
            vocab_size: 8000,
            hidden_size: 128,
            num_layers: 2,
            num_heads: 4,
            max_seq_len: 128,
        }
    }
}