use crate::error::{ErrorCode, TptrError, TptrResult};
use crate::inference::ModelInfo;
#[derive(Debug, Clone)]
pub enum ForwardOp {
Embed { vocab_size: u32, hidden_dim: u32 },
RmsNorm { hidden_dim: u32, eps: f32 },
Attention {
num_heads: u32,
num_kv_heads: u32,
hidden_dim: u32,
head_dim: u32,
},
FfnSwiGlu { hidden_dim: u32, ffn_dim: u32 },
LinearOut {
hidden_dim: u32,
vocab_size: u32,
tied: bool,
},
Softmax { dim: i32 },
Sampling { temperature: f32, top_k: u32 },
ApplyRope {
head_dim: u32,
base: f32,
pos: usize,
},
}
#[derive(Debug, Clone)]
pub struct ArchTemplate {
pub num_layers: u32,
pub pre_ops: Vec<ForwardOp>,
pub per_layer_ops: Vec<ForwardOp>,
pub post_ops: Vec<ForwardOp>,
}
pub fn template_for_arch(arch: &str, info: &ModelInfo) -> TptrResult<ArchTemplate> {
match arch {
"llama3" | "llama" => Ok(llama3(info)),
"mistral" => Ok(mistral(info)),
"qwen2" | "qwen" => Ok(qwen2(info)),
"phi3" => Ok(phi3(info)),
"gemma2" | "gemma" => Ok(gemma2(info)),
other => Err(TptrError::new(
ErrorCode::UnsupportedFeature,
format!(
"unsupported model architecture '{}'; supported: llama3, mistral, qwen2, phi3, gemma2",
other
),
)),
}
}
fn llama3(info: &ModelInfo) -> ArchTemplate {
let num_heads = info.num_heads.max(1);
let head_dim = info.hidden_dim / num_heads;
let num_kv_heads = if info.num_kv_heads > 0 {
info.num_kv_heads
} else {
num_heads / 4
};
ArchTemplate {
num_layers: info.num_layers,
pre_ops: vec![ForwardOp::Embed {
vocab_size: info.vocab_size,
hidden_dim: info.hidden_dim,
}],
per_layer_ops: vec![
ForwardOp::RmsNorm {
hidden_dim: info.hidden_dim,
eps: 1e-5,
},
ForwardOp::ApplyRope {
head_dim,
base: 500_000.0,
pos: 0,
},
ForwardOp::Attention {
num_heads,
num_kv_heads,
hidden_dim: info.hidden_dim,
head_dim,
},
ForwardOp::RmsNorm {
hidden_dim: info.hidden_dim,
eps: 1e-5,
},
ForwardOp::FfnSwiGlu {
hidden_dim: info.hidden_dim,
ffn_dim: info.ffn_dim,
},
],
post_ops: vec![
ForwardOp::RmsNorm {
hidden_dim: info.hidden_dim,
eps: 1e-5,
},
ForwardOp::LinearOut {
hidden_dim: info.hidden_dim,
vocab_size: info.vocab_size,
tied: false,
},
ForwardOp::Softmax { dim: -1 },
ForwardOp::Sampling {
temperature: 1.0,
top_k: 1,
},
],
}
}
fn mistral(info: &ModelInfo) -> ArchTemplate {
let num_heads = info.num_heads.max(1);
let head_dim = info.hidden_dim / num_heads;
let num_kv_heads = if info.num_kv_heads > 0 {
info.num_kv_heads
} else {
8
};
ArchTemplate {
num_layers: info.num_layers,
pre_ops: vec![ForwardOp::Embed {
vocab_size: info.vocab_size,
hidden_dim: info.hidden_dim,
}],
per_layer_ops: vec![
ForwardOp::RmsNorm {
hidden_dim: info.hidden_dim,
eps: 1e-5,
},
ForwardOp::ApplyRope {
head_dim,
base: 10_000.0,
pos: 0,
},
ForwardOp::Attention {
num_heads,
num_kv_heads,
hidden_dim: info.hidden_dim,
head_dim,
},
ForwardOp::RmsNorm {
hidden_dim: info.hidden_dim,
eps: 1e-5,
},
ForwardOp::FfnSwiGlu {
hidden_dim: info.hidden_dim,
ffn_dim: info.ffn_dim,
},
],
post_ops: vec![
ForwardOp::RmsNorm {
hidden_dim: info.hidden_dim,
eps: 1e-5,
},
ForwardOp::LinearOut {
hidden_dim: info.hidden_dim,
vocab_size: info.vocab_size,
tied: false,
},
ForwardOp::Softmax { dim: -1 },
ForwardOp::Sampling {
temperature: 1.0,
top_k: 1,
},
],
}
}
fn qwen2(info: &ModelInfo) -> ArchTemplate {
let num_heads = info.num_heads.max(1);
let head_dim = info.hidden_dim / num_heads;
let num_kv_heads = if info.num_kv_heads > 0 {
info.num_kv_heads
} else {
4
};
ArchTemplate {
num_layers: info.num_layers,
pre_ops: vec![ForwardOp::Embed {
vocab_size: info.vocab_size,
hidden_dim: info.hidden_dim,
}],
per_layer_ops: vec![
ForwardOp::RmsNorm {
hidden_dim: info.hidden_dim,
eps: 1e-6,
},
ForwardOp::ApplyRope {
head_dim,
base: 1_000_000.0,
pos: 0,
},
ForwardOp::Attention {
num_heads,
num_kv_heads,
hidden_dim: info.hidden_dim,
head_dim,
},
ForwardOp::RmsNorm {
hidden_dim: info.hidden_dim,
eps: 1e-6,
},
ForwardOp::FfnSwiGlu {
hidden_dim: info.hidden_dim,
ffn_dim: info.ffn_dim,
},
],
post_ops: vec![
ForwardOp::RmsNorm {
hidden_dim: info.hidden_dim,
eps: 1e-6,
},
ForwardOp::LinearOut {
hidden_dim: info.hidden_dim,
vocab_size: info.vocab_size,
tied: true,
},
ForwardOp::Softmax { dim: -1 },
ForwardOp::Sampling {
temperature: 1.0,
top_k: 1,
},
],
}
}
fn phi3(info: &ModelInfo) -> ArchTemplate {
let num_heads = info.num_heads.max(1);
let head_dim = info.hidden_dim / num_heads;
let num_kv_heads = if info.num_kv_heads > 0 {
info.num_kv_heads
} else {
num_heads
};
let ffn_dim = if info.ffn_dim > 0 {
info.ffn_dim
} else {
info.hidden_dim * 3
};
ArchTemplate {
num_layers: info.num_layers,
pre_ops: vec![ForwardOp::Embed {
vocab_size: info.vocab_size,
hidden_dim: info.hidden_dim,
}],
per_layer_ops: vec![
ForwardOp::RmsNorm {
hidden_dim: info.hidden_dim,
eps: 1e-5,
},
ForwardOp::ApplyRope {
head_dim,
base: 10_000.0,
pos: 0,
},
ForwardOp::Attention {
num_heads,
num_kv_heads,
hidden_dim: info.hidden_dim,
head_dim,
},
ForwardOp::RmsNorm {
hidden_dim: info.hidden_dim,
eps: 1e-5,
},
ForwardOp::FfnSwiGlu {
hidden_dim: info.hidden_dim,
ffn_dim,
},
],
post_ops: vec![
ForwardOp::RmsNorm {
hidden_dim: info.hidden_dim,
eps: 1e-5,
},
ForwardOp::LinearOut {
hidden_dim: info.hidden_dim,
vocab_size: info.vocab_size,
tied: false,
},
ForwardOp::Softmax { dim: -1 },
ForwardOp::Sampling {
temperature: 1.0,
top_k: 1,
},
],
}
}
fn gemma2(info: &ModelInfo) -> ArchTemplate {
let num_heads = info.num_heads.max(1);
let head_dim = info.hidden_dim / num_heads;
let num_kv_heads = if info.num_kv_heads > 0 {
info.num_kv_heads
} else {
num_heads / 2
};
ArchTemplate {
num_layers: info.num_layers,
pre_ops: vec![ForwardOp::Embed {
vocab_size: info.vocab_size,
hidden_dim: info.hidden_dim,
}],
per_layer_ops: vec![
ForwardOp::RmsNorm {
hidden_dim: info.hidden_dim,
eps: 1e-6,
}, ForwardOp::ApplyRope {
head_dim,
base: 10_000.0,
pos: 0,
},
ForwardOp::Attention {
num_heads,
num_kv_heads,
hidden_dim: info.hidden_dim,
head_dim,
},
ForwardOp::RmsNorm {
hidden_dim: info.hidden_dim,
eps: 1e-6,
}, ForwardOp::RmsNorm {
hidden_dim: info.hidden_dim,
eps: 1e-6,
}, ForwardOp::FfnSwiGlu {
hidden_dim: info.hidden_dim,
ffn_dim: info.ffn_dim,
},
ForwardOp::RmsNorm {
hidden_dim: info.hidden_dim,
eps: 1e-6,
}, ],
post_ops: vec![
ForwardOp::RmsNorm {
hidden_dim: info.hidden_dim,
eps: 1e-6,
},
ForwardOp::LinearOut {
hidden_dim: info.hidden_dim,
vocab_size: info.vocab_size,
tied: true,
},
ForwardOp::Softmax { dim: -1 },
ForwardOp::Sampling {
temperature: 1.0,
top_k: 1,
},
],
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::inference::ModelInfo;
use std::path::PathBuf;
fn stub_info(arch: &str, num_layers: u32) -> ModelInfo {
ModelInfo {
path: PathBuf::from("stub.gguf"),
arch: arch.to_string(),
context_len: 4096,
vocab_size: 32000,
hidden_dim: 4096,
num_heads: 32,
num_kv_heads: 8,
ffn_dim: 11008,
num_layers,
per_layer_bits: Vec::new(),
pruning_mask: None,
}
}
#[test]
fn llama3_layer_count() {
let info = stub_info("llama3", 32);
let t = template_for_arch("llama3", &info).unwrap();
assert_eq!(t.num_layers, 32);
assert_eq!(t.per_layer_ops.len(), 5);
}
#[test]
fn qwen2_tied_lm_head() {
let info = stub_info("qwen2", 28);
let t = template_for_arch("qwen2", &info).unwrap();
let tied = t
.post_ops
.iter()
.any(|op| matches!(op, ForwardOp::LinearOut { tied: true, .. }));
assert!(tied, "Qwen2 must have tied lm_head");
}
#[test]
fn gemma2_has_seven_per_layer_ops() {
let info = stub_info("gemma2", 46);
let t = template_for_arch("gemma2", &info).unwrap();
assert_eq!(
t.per_layer_ops.len(),
7,
"Gemma2 uses 7 ops/layer (pre-attn norm + RoPE + attn + post-attn norm + pre-FFN norm + FFN + post-FFN norm)"
);
}
#[test]
fn unknown_arch_is_unsupported_feature_error() {
let info = stub_info("gpt2-xl", 48);
let err = template_for_arch("gpt2-xl", &info).unwrap_err();
assert_eq!(err.code, ErrorCode::UnsupportedFeature);
}
#[test]
fn phi3_ffn_fallback_to_3x_hidden() {
let mut info = stub_info("phi3", 32);
info.ffn_dim = 0; let t = template_for_arch("phi3", &info).unwrap();
let ffn = t
.per_layer_ops
.iter()
.find_map(|op| {
if let ForwardOp::FfnSwiGlu {
hidden_dim,
ffn_dim,
} = op
{
Some((*hidden_dim, *ffn_dim))
} else {
None
}
})
.unwrap();
assert_eq!(ffn.1, ffn.0 * 3);
}
#[test]
fn mistral_has_correct_kv_heads_fallback() {
let mut info = stub_info("mistral", 32);
info.num_kv_heads = 0;
let t = template_for_arch("mistral", &info).unwrap();
let kv = t
.per_layer_ops
.iter()
.find_map(|op| {
if let ForwardOp::Attention { num_kv_heads, .. } = op {
Some(*num_kv_heads)
} else {
None
}
})
.unwrap();
assert_eq!(kv, 8);
}
}