1use crate::error::{ErrorCode, TptrError, TptrResult};
9use crate::inference::ModelInfo;
10
11#[derive(Debug, Clone)]
13pub enum ForwardOp {
14 Embed { vocab_size: u32, hidden_dim: u32 },
16 RmsNorm { hidden_dim: u32, eps: f32 },
18 Attention {
20 num_heads: u32,
21 num_kv_heads: u32,
22 hidden_dim: u32,
23 head_dim: u32,
24 },
25 FfnSwiGlu { hidden_dim: u32, ffn_dim: u32 },
27 LinearOut {
30 hidden_dim: u32,
31 vocab_size: u32,
32 tied: bool,
33 },
34 Softmax { dim: i32 },
36 Sampling { temperature: f32, top_k: u32 },
38 ApplyRope {
43 head_dim: u32,
44 base: f32,
45 pos: usize,
46 },
47}
48
49#[derive(Debug, Clone)]
51pub struct ArchTemplate {
52 pub num_layers: u32,
54 pub pre_ops: Vec<ForwardOp>,
56 pub per_layer_ops: Vec<ForwardOp>,
58 pub post_ops: Vec<ForwardOp>,
60}
61
62pub fn template_for_arch(arch: &str, info: &ModelInfo) -> TptrResult<ArchTemplate> {
66 match arch {
67 "llama3" | "llama" => Ok(llama3(info)),
68 "mistral" => Ok(mistral(info)),
69 "qwen2" | "qwen" => Ok(qwen2(info)),
70 "phi3" => Ok(phi3(info)),
71 "gemma2" | "gemma" => Ok(gemma2(info)),
72 other => Err(TptrError::new(
73 ErrorCode::UnsupportedFeature,
74 format!(
75 "unsupported model architecture '{}'; supported: llama3, mistral, qwen2, phi3, gemma2",
76 other
77 ),
78 )),
79 }
80}
81
82fn llama3(info: &ModelInfo) -> ArchTemplate {
87 let num_heads = info.num_heads.max(1);
88 let head_dim = info.hidden_dim / num_heads;
89 let num_kv_heads = if info.num_kv_heads > 0 {
90 info.num_kv_heads
91 } else {
92 num_heads / 4
93 };
94 ArchTemplate {
95 num_layers: info.num_layers,
96 pre_ops: vec![ForwardOp::Embed {
97 vocab_size: info.vocab_size,
98 hidden_dim: info.hidden_dim,
99 }],
100 per_layer_ops: vec![
101 ForwardOp::RmsNorm {
102 hidden_dim: info.hidden_dim,
103 eps: 1e-5,
104 },
105 ForwardOp::ApplyRope {
106 head_dim,
107 base: 500_000.0,
108 pos: 0,
109 },
110 ForwardOp::Attention {
111 num_heads,
112 num_kv_heads,
113 hidden_dim: info.hidden_dim,
114 head_dim,
115 },
116 ForwardOp::RmsNorm {
117 hidden_dim: info.hidden_dim,
118 eps: 1e-5,
119 },
120 ForwardOp::FfnSwiGlu {
121 hidden_dim: info.hidden_dim,
122 ffn_dim: info.ffn_dim,
123 },
124 ],
125 post_ops: vec![
126 ForwardOp::RmsNorm {
127 hidden_dim: info.hidden_dim,
128 eps: 1e-5,
129 },
130 ForwardOp::LinearOut {
131 hidden_dim: info.hidden_dim,
132 vocab_size: info.vocab_size,
133 tied: false,
134 },
135 ForwardOp::Softmax { dim: -1 },
136 ForwardOp::Sampling {
137 temperature: 1.0,
138 top_k: 1,
139 },
140 ],
141 }
142}
143
144fn mistral(info: &ModelInfo) -> ArchTemplate {
145 let num_heads = info.num_heads.max(1);
146 let head_dim = info.hidden_dim / num_heads;
147 let num_kv_heads = if info.num_kv_heads > 0 {
149 info.num_kv_heads
150 } else {
151 8
152 };
153 ArchTemplate {
154 num_layers: info.num_layers,
155 pre_ops: vec![ForwardOp::Embed {
156 vocab_size: info.vocab_size,
157 hidden_dim: info.hidden_dim,
158 }],
159 per_layer_ops: vec![
160 ForwardOp::RmsNorm {
161 hidden_dim: info.hidden_dim,
162 eps: 1e-5,
163 },
164 ForwardOp::ApplyRope {
165 head_dim,
166 base: 10_000.0,
167 pos: 0,
168 },
169 ForwardOp::Attention {
170 num_heads,
171 num_kv_heads,
172 hidden_dim: info.hidden_dim,
173 head_dim,
174 },
175 ForwardOp::RmsNorm {
176 hidden_dim: info.hidden_dim,
177 eps: 1e-5,
178 },
179 ForwardOp::FfnSwiGlu {
180 hidden_dim: info.hidden_dim,
181 ffn_dim: info.ffn_dim,
182 },
183 ],
184 post_ops: vec![
185 ForwardOp::RmsNorm {
186 hidden_dim: info.hidden_dim,
187 eps: 1e-5,
188 },
189 ForwardOp::LinearOut {
190 hidden_dim: info.hidden_dim,
191 vocab_size: info.vocab_size,
192 tied: false,
193 },
194 ForwardOp::Softmax { dim: -1 },
195 ForwardOp::Sampling {
196 temperature: 1.0,
197 top_k: 1,
198 },
199 ],
200 }
201}
202
203fn qwen2(info: &ModelInfo) -> ArchTemplate {
204 let num_heads = info.num_heads.max(1);
205 let head_dim = info.hidden_dim / num_heads;
206 let num_kv_heads = if info.num_kv_heads > 0 {
208 info.num_kv_heads
209 } else {
210 4
211 };
212 ArchTemplate {
213 num_layers: info.num_layers,
214 pre_ops: vec![ForwardOp::Embed {
215 vocab_size: info.vocab_size,
216 hidden_dim: info.hidden_dim,
217 }],
218 per_layer_ops: vec![
219 ForwardOp::RmsNorm {
221 hidden_dim: info.hidden_dim,
222 eps: 1e-6,
223 },
224 ForwardOp::ApplyRope {
225 head_dim,
226 base: 1_000_000.0,
227 pos: 0,
228 },
229 ForwardOp::Attention {
230 num_heads,
231 num_kv_heads,
232 hidden_dim: info.hidden_dim,
233 head_dim,
234 },
235 ForwardOp::RmsNorm {
236 hidden_dim: info.hidden_dim,
237 eps: 1e-6,
238 },
239 ForwardOp::FfnSwiGlu {
240 hidden_dim: info.hidden_dim,
241 ffn_dim: info.ffn_dim,
242 },
243 ],
244 post_ops: vec![
246 ForwardOp::RmsNorm {
247 hidden_dim: info.hidden_dim,
248 eps: 1e-6,
249 },
250 ForwardOp::LinearOut {
251 hidden_dim: info.hidden_dim,
252 vocab_size: info.vocab_size,
253 tied: true,
254 },
255 ForwardOp::Softmax { dim: -1 },
256 ForwardOp::Sampling {
257 temperature: 1.0,
258 top_k: 1,
259 },
260 ],
261 }
262}
263
264fn phi3(info: &ModelInfo) -> ArchTemplate {
265 let num_heads = info.num_heads.max(1);
266 let head_dim = info.hidden_dim / num_heads;
267 let num_kv_heads = if info.num_kv_heads > 0 {
268 info.num_kv_heads
269 } else {
270 num_heads
271 };
272 let ffn_dim = if info.ffn_dim > 0 {
274 info.ffn_dim
275 } else {
276 info.hidden_dim * 3
277 };
278 ArchTemplate {
279 num_layers: info.num_layers,
280 pre_ops: vec![ForwardOp::Embed {
281 vocab_size: info.vocab_size,
282 hidden_dim: info.hidden_dim,
283 }],
284 per_layer_ops: vec![
285 ForwardOp::RmsNorm {
286 hidden_dim: info.hidden_dim,
287 eps: 1e-5,
288 },
289 ForwardOp::ApplyRope {
290 head_dim,
291 base: 10_000.0,
292 pos: 0,
293 },
294 ForwardOp::Attention {
295 num_heads,
296 num_kv_heads,
297 hidden_dim: info.hidden_dim,
298 head_dim,
299 },
300 ForwardOp::RmsNorm {
301 hidden_dim: info.hidden_dim,
302 eps: 1e-5,
303 },
304 ForwardOp::FfnSwiGlu {
305 hidden_dim: info.hidden_dim,
306 ffn_dim,
307 },
308 ],
309 post_ops: vec![
310 ForwardOp::RmsNorm {
311 hidden_dim: info.hidden_dim,
312 eps: 1e-5,
313 },
314 ForwardOp::LinearOut {
315 hidden_dim: info.hidden_dim,
316 vocab_size: info.vocab_size,
317 tied: false,
318 },
319 ForwardOp::Softmax { dim: -1 },
320 ForwardOp::Sampling {
321 temperature: 1.0,
322 top_k: 1,
323 },
324 ],
325 }
326}
327
328fn gemma2(info: &ModelInfo) -> ArchTemplate {
329 let num_heads = info.num_heads.max(1);
330 let head_dim = info.hidden_dim / num_heads;
331 let num_kv_heads = if info.num_kv_heads > 0 {
332 info.num_kv_heads
333 } else {
334 num_heads / 2
335 };
336 ArchTemplate {
338 num_layers: info.num_layers,
339 pre_ops: vec![ForwardOp::Embed {
340 vocab_size: info.vocab_size,
341 hidden_dim: info.hidden_dim,
342 }],
343 per_layer_ops: vec![
344 ForwardOp::RmsNorm {
345 hidden_dim: info.hidden_dim,
346 eps: 1e-6,
347 }, ForwardOp::ApplyRope {
349 head_dim,
350 base: 10_000.0,
351 pos: 0,
352 },
353 ForwardOp::Attention {
354 num_heads,
355 num_kv_heads,
356 hidden_dim: info.hidden_dim,
357 head_dim,
358 },
359 ForwardOp::RmsNorm {
360 hidden_dim: info.hidden_dim,
361 eps: 1e-6,
362 }, ForwardOp::RmsNorm {
364 hidden_dim: info.hidden_dim,
365 eps: 1e-6,
366 }, ForwardOp::FfnSwiGlu {
368 hidden_dim: info.hidden_dim,
369 ffn_dim: info.ffn_dim,
370 },
371 ForwardOp::RmsNorm {
372 hidden_dim: info.hidden_dim,
373 eps: 1e-6,
374 }, ],
376 post_ops: vec![
378 ForwardOp::RmsNorm {
379 hidden_dim: info.hidden_dim,
380 eps: 1e-6,
381 },
382 ForwardOp::LinearOut {
383 hidden_dim: info.hidden_dim,
384 vocab_size: info.vocab_size,
385 tied: true,
386 },
387 ForwardOp::Softmax { dim: -1 },
388 ForwardOp::Sampling {
389 temperature: 1.0,
390 top_k: 1,
391 },
392 ],
393 }
394}
395
396#[cfg(test)]
401mod tests {
402 use super::*;
403 use crate::inference::ModelInfo;
404 use std::path::PathBuf;
405
406 fn stub_info(arch: &str, num_layers: u32) -> ModelInfo {
407 ModelInfo {
408 path: PathBuf::from("stub.gguf"),
409 arch: arch.to_string(),
410 context_len: 4096,
411 vocab_size: 32000,
412 hidden_dim: 4096,
413 num_heads: 32,
414 num_kv_heads: 8,
415 ffn_dim: 11008,
416 num_layers,
417 per_layer_bits: Vec::new(),
418 pruning_mask: None,
419 }
420 }
421
422 #[test]
423 fn llama3_layer_count() {
424 let info = stub_info("llama3", 32);
425 let t = template_for_arch("llama3", &info).unwrap();
426 assert_eq!(t.num_layers, 32);
427 assert_eq!(t.per_layer_ops.len(), 5);
429 }
430
431 #[test]
432 fn qwen2_tied_lm_head() {
433 let info = stub_info("qwen2", 28);
434 let t = template_for_arch("qwen2", &info).unwrap();
435 let tied = t
436 .post_ops
437 .iter()
438 .any(|op| matches!(op, ForwardOp::LinearOut { tied: true, .. }));
439 assert!(tied, "Qwen2 must have tied lm_head");
440 }
441
442 #[test]
443 fn gemma2_has_seven_per_layer_ops() {
444 let info = stub_info("gemma2", 46);
445 let t = template_for_arch("gemma2", &info).unwrap();
446 assert_eq!(
447 t.per_layer_ops.len(),
448 7,
449 "Gemma2 uses 7 ops/layer (pre-attn norm + RoPE + attn + post-attn norm + pre-FFN norm + FFN + post-FFN norm)"
450 );
451 }
452
453 #[test]
454 fn unknown_arch_is_unsupported_feature_error() {
455 let info = stub_info("gpt2-xl", 48);
456 let err = template_for_arch("gpt2-xl", &info).unwrap_err();
457 assert_eq!(err.code, ErrorCode::UnsupportedFeature);
458 }
459
460 #[test]
461 fn phi3_ffn_fallback_to_3x_hidden() {
462 let mut info = stub_info("phi3", 32);
463 info.ffn_dim = 0; let t = template_for_arch("phi3", &info).unwrap();
465 let ffn = t
466 .per_layer_ops
467 .iter()
468 .find_map(|op| {
469 if let ForwardOp::FfnSwiGlu {
470 hidden_dim,
471 ffn_dim,
472 } = op
473 {
474 Some((*hidden_dim, *ffn_dim))
475 } else {
476 None
477 }
478 })
479 .unwrap();
480 assert_eq!(ffn.1, ffn.0 * 3);
481 }
482
483 #[test]
484 fn mistral_has_correct_kv_heads_fallback() {
485 let mut info = stub_info("mistral", 32);
486 info.num_kv_heads = 0;
487 let t = template_for_arch("mistral", &info).unwrap();
488 let kv = t
489 .per_layer_ops
490 .iter()
491 .find_map(|op| {
492 if let ForwardOp::Attention { num_kv_heads, .. } = op {
493 Some(*num_kv_heads)
494 } else {
495 None
496 }
497 })
498 .unwrap();
499 assert_eq!(kv, 8);
500 }
501}