Model Architecture Language (MAL) for Hermes LLM

A composable DSL for defining LLM model architectures using pest parser.

Example MAL - Simple (flat) style

model tiny {
    vocab_size: 32000
    hidden_size: 128
    num_layers: 4
    num_heads: 4
    intermediate_size: 512
}

Example MAL - Composable style

# Define attention mechanism
attention gqa {
    num_heads: 32
    num_kv_heads: 8
    head_dim: 128
    position_encoding: rope { theta: 10000.0 }
}

# Define FFN
ffn swiglu_mlp {
    hidden_dim: 14336
    activation: swiglu
    bias: false
}

# Define transformer block
block llama_block {
    attention: gqa
    ffn: swiglu_mlp
    norm: rmsnorm { eps: 1e-5 }
    norm_position: pre
}

# Define model using the block
model llama_7b {
    vocab_size: 32000
    max_seq_len: 4096
    hidden_size: 4096
    block: llama_block
    num_layers: 32
}