Skip to main content

tpt_gpu_runtime/
arch.rs

1//! Arch-template dispatch layer.
2//!
3//! Maps a model architecture string (from the GGUF `general.architecture` key)
4//! to a typed sequence of `ForwardOp`s that fully describes the model's
5//! forward pass. Adding a new architecture requires one new function and one
6//! new match arm in `template_for_arch` — no changes to the dispatch loop.
7
8use crate::error::{ErrorCode, TptrError, TptrResult};
9use crate::inference::ModelInfo;
10
11/// A single operation in the transformer forward pass.
12#[derive(Debug, Clone)]
13pub enum ForwardOp {
14    /// Token → hidden state lookup via an embedding table.
15    Embed { vocab_size: u32, hidden_dim: u32 },
16    /// RMS Layer Normalisation: y = x / rms(x) * gamma.
17    RmsNorm { hidden_dim: u32, eps: f32 },
18    /// Scaled dot-product attention with optional grouped-query (GQA).
19    Attention {
20        num_heads: u32,
21        num_kv_heads: u32,
22        hidden_dim: u32,
23        head_dim: u32,
24    },
25    /// SwiGLU feed-forward block: FFN(x) = down(silu(gate(x)) * up(x)).
26    FfnSwiGlu { hidden_dim: u32, ffn_dim: u32 },
27    /// Final vocabulary projection (lm_head). `tied = true` for Qwen2/Gemma2
28    /// where the embedding table and lm_head share weights.
29    LinearOut {
30        hidden_dim: u32,
31        vocab_size: u32,
32        tied: bool,
33    },
34    /// Softmax over the vocab dimension.
35    Softmax { dim: i32 },
36    /// Sample the next token from a probability distribution.
37    Sampling { temperature: f32, top_k: u32 },
38    /// Apply Rotary Position Embedding to Q and K projections.
39    ///
40    /// `pos` is a placeholder (0) in the template; the runtime substitutes the
41    /// real decode position from `kv_cache.seq_len` at forward-pass time.
42    ApplyRope {
43        head_dim: u32,
44        base: f32,
45        pos: usize,
46    },
47}
48
49/// The complete op sequence for one model architecture.
50#[derive(Debug, Clone)]
51pub struct ArchTemplate {
52    /// Number of transformer layers (blocks).
53    pub num_layers: u32,
54    /// Ops run once before the first layer (typically: Embed).
55    pub pre_ops: Vec<ForwardOp>,
56    /// Ops run once per transformer layer, repeated `num_layers` times.
57    pub per_layer_ops: Vec<ForwardOp>,
58    /// Ops run once after the last layer (final norm, lm_head, softmax, sample).
59    pub post_ops: Vec<ForwardOp>,
60}
61
62/// Build an `ArchTemplate` from the `general.architecture` GGUF key.
63///
64/// Returns `UnsupportedFeature` for unrecognised architectures.
65pub fn template_for_arch(arch: &str, info: &ModelInfo) -> TptrResult<ArchTemplate> {
66    match arch {
67        "llama3" | "llama" => Ok(llama3(info)),
68        "mistral"          => Ok(mistral(info)),
69        "qwen2" | "qwen"   => Ok(qwen2(info)),
70        "phi3"             => Ok(phi3(info)),
71        "gemma2" | "gemma" => Ok(gemma2(info)),
72        other => Err(TptrError::new(
73            ErrorCode::UnsupportedFeature,
74            format!(
75                "unsupported model architecture '{}'; supported: llama3, mistral, qwen2, phi3, gemma2",
76                other
77            ),
78        )),
79    }
80}
81
82// ---------------------------------------------------------------------------
83// Arch implementations
84// ---------------------------------------------------------------------------
85
86fn llama3(info: &ModelInfo) -> ArchTemplate {
87    let num_heads = info.num_heads.max(1);
88    let head_dim = info.hidden_dim / num_heads;
89    let num_kv_heads = if info.num_kv_heads > 0 {
90        info.num_kv_heads
91    } else {
92        num_heads / 4
93    };
94    ArchTemplate {
95        num_layers: info.num_layers,
96        pre_ops: vec![ForwardOp::Embed {
97            vocab_size: info.vocab_size,
98            hidden_dim: info.hidden_dim,
99        }],
100        per_layer_ops: vec![
101            ForwardOp::RmsNorm {
102                hidden_dim: info.hidden_dim,
103                eps: 1e-5,
104            },
105            ForwardOp::ApplyRope {
106                head_dim,
107                base: 500_000.0,
108                pos: 0,
109            },
110            ForwardOp::Attention {
111                num_heads,
112                num_kv_heads,
113                hidden_dim: info.hidden_dim,
114                head_dim,
115            },
116            ForwardOp::RmsNorm {
117                hidden_dim: info.hidden_dim,
118                eps: 1e-5,
119            },
120            ForwardOp::FfnSwiGlu {
121                hidden_dim: info.hidden_dim,
122                ffn_dim: info.ffn_dim,
123            },
124        ],
125        post_ops: vec![
126            ForwardOp::RmsNorm {
127                hidden_dim: info.hidden_dim,
128                eps: 1e-5,
129            },
130            ForwardOp::LinearOut {
131                hidden_dim: info.hidden_dim,
132                vocab_size: info.vocab_size,
133                tied: false,
134            },
135            ForwardOp::Softmax { dim: -1 },
136            ForwardOp::Sampling {
137                temperature: 1.0,
138                top_k: 1,
139            },
140        ],
141    }
142}
143
144fn mistral(info: &ModelInfo) -> ArchTemplate {
145    let num_heads = info.num_heads.max(1);
146    let head_dim = info.hidden_dim / num_heads;
147    // Mistral 7B uses 8 KV heads by default; fall back if not in GGUF.
148    let num_kv_heads = if info.num_kv_heads > 0 {
149        info.num_kv_heads
150    } else {
151        8
152    };
153    ArchTemplate {
154        num_layers: info.num_layers,
155        pre_ops: vec![ForwardOp::Embed {
156            vocab_size: info.vocab_size,
157            hidden_dim: info.hidden_dim,
158        }],
159        per_layer_ops: vec![
160            ForwardOp::RmsNorm {
161                hidden_dim: info.hidden_dim,
162                eps: 1e-5,
163            },
164            ForwardOp::ApplyRope {
165                head_dim,
166                base: 10_000.0,
167                pos: 0,
168            },
169            ForwardOp::Attention {
170                num_heads,
171                num_kv_heads,
172                hidden_dim: info.hidden_dim,
173                head_dim,
174            },
175            ForwardOp::RmsNorm {
176                hidden_dim: info.hidden_dim,
177                eps: 1e-5,
178            },
179            ForwardOp::FfnSwiGlu {
180                hidden_dim: info.hidden_dim,
181                ffn_dim: info.ffn_dim,
182            },
183        ],
184        post_ops: vec![
185            ForwardOp::RmsNorm {
186                hidden_dim: info.hidden_dim,
187                eps: 1e-5,
188            },
189            ForwardOp::LinearOut {
190                hidden_dim: info.hidden_dim,
191                vocab_size: info.vocab_size,
192                tied: false,
193            },
194            ForwardOp::Softmax { dim: -1 },
195            ForwardOp::Sampling {
196                temperature: 1.0,
197                top_k: 1,
198            },
199        ],
200    }
201}
202
203fn qwen2(info: &ModelInfo) -> ArchTemplate {
204    let num_heads = info.num_heads.max(1);
205    let head_dim = info.hidden_dim / num_heads;
206    // Qwen2-7B uses 4 KV heads; 72B uses 8.
207    let num_kv_heads = if info.num_kv_heads > 0 {
208        info.num_kv_heads
209    } else {
210        4
211    };
212    ArchTemplate {
213        num_layers: info.num_layers,
214        pre_ops: vec![ForwardOp::Embed {
215            vocab_size: info.vocab_size,
216            hidden_dim: info.hidden_dim,
217        }],
218        per_layer_ops: vec![
219            // Qwen2 uses eps = 1e-6 (tighter than Llama3/Mistral)
220            ForwardOp::RmsNorm {
221                hidden_dim: info.hidden_dim,
222                eps: 1e-6,
223            },
224            ForwardOp::ApplyRope {
225                head_dim,
226                base: 1_000_000.0,
227                pos: 0,
228            },
229            ForwardOp::Attention {
230                num_heads,
231                num_kv_heads,
232                hidden_dim: info.hidden_dim,
233                head_dim,
234            },
235            ForwardOp::RmsNorm {
236                hidden_dim: info.hidden_dim,
237                eps: 1e-6,
238            },
239            ForwardOp::FfnSwiGlu {
240                hidden_dim: info.hidden_dim,
241                ffn_dim: info.ffn_dim,
242            },
243        ],
244        // Qwen2 ties input embeddings to lm_head (vocab_size = 151936 typically)
245        post_ops: vec![
246            ForwardOp::RmsNorm {
247                hidden_dim: info.hidden_dim,
248                eps: 1e-6,
249            },
250            ForwardOp::LinearOut {
251                hidden_dim: info.hidden_dim,
252                vocab_size: info.vocab_size,
253                tied: true,
254            },
255            ForwardOp::Softmax { dim: -1 },
256            ForwardOp::Sampling {
257                temperature: 1.0,
258                top_k: 1,
259            },
260        ],
261    }
262}
263
264fn phi3(info: &ModelInfo) -> ArchTemplate {
265    let num_heads = info.num_heads.max(1);
266    let head_dim = info.hidden_dim / num_heads;
267    let num_kv_heads = if info.num_kv_heads > 0 {
268        info.num_kv_heads
269    } else {
270        num_heads
271    };
272    // Phi-3 uses ffn_dim = 3× hidden_dim when not specified in GGUF.
273    let ffn_dim = if info.ffn_dim > 0 {
274        info.ffn_dim
275    } else {
276        info.hidden_dim * 3
277    };
278    ArchTemplate {
279        num_layers: info.num_layers,
280        pre_ops: vec![ForwardOp::Embed {
281            vocab_size: info.vocab_size,
282            hidden_dim: info.hidden_dim,
283        }],
284        per_layer_ops: vec![
285            ForwardOp::RmsNorm {
286                hidden_dim: info.hidden_dim,
287                eps: 1e-5,
288            },
289            ForwardOp::ApplyRope {
290                head_dim,
291                base: 10_000.0,
292                pos: 0,
293            },
294            ForwardOp::Attention {
295                num_heads,
296                num_kv_heads,
297                hidden_dim: info.hidden_dim,
298                head_dim,
299            },
300            ForwardOp::RmsNorm {
301                hidden_dim: info.hidden_dim,
302                eps: 1e-5,
303            },
304            ForwardOp::FfnSwiGlu {
305                hidden_dim: info.hidden_dim,
306                ffn_dim,
307            },
308        ],
309        post_ops: vec![
310            ForwardOp::RmsNorm {
311                hidden_dim: info.hidden_dim,
312                eps: 1e-5,
313            },
314            ForwardOp::LinearOut {
315                hidden_dim: info.hidden_dim,
316                vocab_size: info.vocab_size,
317                tied: false,
318            },
319            ForwardOp::Softmax { dim: -1 },
320            ForwardOp::Sampling {
321                temperature: 1.0,
322                top_k: 1,
323            },
324        ],
325    }
326}
327
328fn gemma2(info: &ModelInfo) -> ArchTemplate {
329    let num_heads = info.num_heads.max(1);
330    let head_dim = info.hidden_dim / num_heads;
331    let num_kv_heads = if info.num_kv_heads > 0 {
332        info.num_kv_heads
333    } else {
334        num_heads / 2
335    };
336    // Gemma2 adds a post-attention norm and a post-FFN norm (7 ops per layer with RoPE).
337    ArchTemplate {
338        num_layers: info.num_layers,
339        pre_ops: vec![ForwardOp::Embed {
340            vocab_size: info.vocab_size,
341            hidden_dim: info.hidden_dim,
342        }],
343        per_layer_ops: vec![
344            ForwardOp::RmsNorm {
345                hidden_dim: info.hidden_dim,
346                eps: 1e-6,
347            }, // pre-attention
348            ForwardOp::ApplyRope {
349                head_dim,
350                base: 10_000.0,
351                pos: 0,
352            },
353            ForwardOp::Attention {
354                num_heads,
355                num_kv_heads,
356                hidden_dim: info.hidden_dim,
357                head_dim,
358            },
359            ForwardOp::RmsNorm {
360                hidden_dim: info.hidden_dim,
361                eps: 1e-6,
362            }, // post-attention
363            ForwardOp::RmsNorm {
364                hidden_dim: info.hidden_dim,
365                eps: 1e-6,
366            }, // pre-FFN
367            ForwardOp::FfnSwiGlu {
368                hidden_dim: info.hidden_dim,
369                ffn_dim: info.ffn_dim,
370            },
371            ForwardOp::RmsNorm {
372                hidden_dim: info.hidden_dim,
373                eps: 1e-6,
374            }, // post-FFN
375        ],
376        // Gemma2 also ties lm_head with the embedding table.
377        post_ops: vec![
378            ForwardOp::RmsNorm {
379                hidden_dim: info.hidden_dim,
380                eps: 1e-6,
381            },
382            ForwardOp::LinearOut {
383                hidden_dim: info.hidden_dim,
384                vocab_size: info.vocab_size,
385                tied: true,
386            },
387            ForwardOp::Softmax { dim: -1 },
388            ForwardOp::Sampling {
389                temperature: 1.0,
390                top_k: 1,
391            },
392        ],
393    }
394}
395
396// ---------------------------------------------------------------------------
397// Tests
398// ---------------------------------------------------------------------------
399
400#[cfg(test)]
401mod tests {
402    use super::*;
403    use crate::inference::ModelInfo;
404    use std::path::PathBuf;
405
406    fn stub_info(arch: &str, num_layers: u32) -> ModelInfo {
407        ModelInfo {
408            path: PathBuf::from("stub.gguf"),
409            arch: arch.to_string(),
410            context_len: 4096,
411            vocab_size: 32000,
412            hidden_dim: 4096,
413            num_heads: 32,
414            num_kv_heads: 8,
415            ffn_dim: 11008,
416            num_layers,
417            per_layer_bits: Vec::new(),
418            pruning_mask: None,
419        }
420    }
421
422    #[test]
423    fn llama3_layer_count() {
424        let info = stub_info("llama3", 32);
425        let t = template_for_arch("llama3", &info).unwrap();
426        assert_eq!(t.num_layers, 32);
427        // RmsNorm + ApplyRope + Attention + RmsNorm + FfnSwiGlu = 5
428        assert_eq!(t.per_layer_ops.len(), 5);
429    }
430
431    #[test]
432    fn qwen2_tied_lm_head() {
433        let info = stub_info("qwen2", 28);
434        let t = template_for_arch("qwen2", &info).unwrap();
435        let tied = t
436            .post_ops
437            .iter()
438            .any(|op| matches!(op, ForwardOp::LinearOut { tied: true, .. }));
439        assert!(tied, "Qwen2 must have tied lm_head");
440    }
441
442    #[test]
443    fn gemma2_has_seven_per_layer_ops() {
444        let info = stub_info("gemma2", 46);
445        let t = template_for_arch("gemma2", &info).unwrap();
446        assert_eq!(
447            t.per_layer_ops.len(),
448            7,
449            "Gemma2 uses 7 ops/layer (pre-attn norm + RoPE + attn + post-attn norm + pre-FFN norm + FFN + post-FFN norm)"
450        );
451    }
452
453    #[test]
454    fn unknown_arch_is_unsupported_feature_error() {
455        let info = stub_info("gpt2-xl", 48);
456        let err = template_for_arch("gpt2-xl", &info).unwrap_err();
457        assert_eq!(err.code, ErrorCode::UnsupportedFeature);
458    }
459
460    #[test]
461    fn phi3_ffn_fallback_to_3x_hidden() {
462        let mut info = stub_info("phi3", 32);
463        info.ffn_dim = 0; // not set in GGUF
464        let t = template_for_arch("phi3", &info).unwrap();
465        let ffn = t
466            .per_layer_ops
467            .iter()
468            .find_map(|op| {
469                if let ForwardOp::FfnSwiGlu {
470                    hidden_dim,
471                    ffn_dim,
472                } = op
473                {
474                    Some((*hidden_dim, *ffn_dim))
475                } else {
476                    None
477                }
478            })
479            .unwrap();
480        assert_eq!(ffn.1, ffn.0 * 3);
481    }
482
483    #[test]
484    fn mistral_has_correct_kv_heads_fallback() {
485        let mut info = stub_info("mistral", 32);
486        info.num_kv_heads = 0;
487        let t = template_for_arch("mistral", &info).unwrap();
488        let kv = t
489            .per_layer_ops
490            .iter()
491            .find_map(|op| {
492                if let ForwardOp::Attention { num_kv_heads, .. } = op {
493                    Some(*num_kv_heads)
494                } else {
495                    None
496                }
497            })
498            .unwrap();
499        assert_eq!(kv, 8);
500    }
501}