use std::collections::HashMap;
use std::io::{Read, Seek, SeekFrom};
use candle_core::quantized::gguf_file;
use candle_core::{DType, Device, Result, Tensor};
use candle_transformers::models::{
quantized_gemma3, quantized_glm4, quantized_lfm2, quantized_llama, quantized_phi,
quantized_phi3, quantized_qwen2, quantized_qwen3, quantized_qwen3_moe,
};
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Architecture {
Llama,
Gemma,
Glm4,
Lfm2,
Phi2,
Phi3,
Qwen2,
Qwen3,
Qwen3Moe,
DeepSeek2,
DeepSeek4,
}
const KNOWN_UNSUPPORTED_ARCHS: &[&str] = &[
"afmoe",
"apertus",
"arcee",
"arctic",
"arwkv7",
"baichuan",
"bailingmoe",
"bailingmoe2",
"bert",
"bitnet",
"bloom",
"chameleon",
"chatglm",
"codeshell",
"cogvlm",
"cohere2",
"command-r",
"dbrx",
"deci",
"deepseek",
"dots1",
"dream",
"ernie4_5",
"ernie4_5-moe",
"exaone",
"exaone4",
"falcon",
"falcon-h1",
"gemma3n",
"glm4moe",
"gpt2",
"gpt-oss",
"gptj",
"gptneox",
"granite",
"granitehybrid",
"granitemoe",
"grok",
"grovemoe",
"hunyuan-dense",
"hunyuan-moe",
"jais",
"jamba",
"jina-bert-v2",
"kimi-k3",
"kimi-linear",
"llada",
"llada-moe",
"llama4",
"lfm2moe",
"mamba",
"mamba2",
"minicpm",
"minicpm3",
"minimax-m2",
"mpt",
"nemotron",
"nemotron-h",
"neo-bert",
"nomic-bert",
"nomic-bert-moe",
"olmo",
"olmo2",
"olmoe",
"openelm",
"orion",
"phimoe",
"plamo",
"plamo2",
"plm",
"qwen",
"qwen2moe",
"qwen2vl",
"refact",
"rwkv6",
"rwkv6qwen2",
"rwkv7",
"seed-oss",
"smallthinker",
"smollm3",
"stablelm",
"starcoder",
"starcoder2",
"t5",
"t5encoder",
"wavtokenizer-dec",
"xverse",
"internlm2",
];
impl Architecture {
pub fn from_name(name: &str) -> Option<Self> {
Some(match name {
"llama" => Self::Llama,
"gemma" | "gemma2" | "gemma3" | "gemma-embedding" => Self::Gemma,
"glm4" => Self::Glm4,
"lfm2" => Self::Lfm2,
"phi2" => Self::Phi2,
"phi3" => Self::Phi3,
"qwen2" => Self::Qwen2,
"qwen3" => Self::Qwen3,
"qwen3moe" => Self::Qwen3Moe,
"deepseek2" => Self::DeepSeek2,
"deepseek4" => Self::DeepSeek4,
_ => return None,
})
}
pub fn from_gguf_metadata(metadata: &HashMap<String, gguf_file::Value>) -> Option<Self> {
Self::from_name(Self::arch_name(metadata)?.as_str())
}
pub fn arch_name(metadata: &HashMap<String, gguf_file::Value>) -> Option<String> {
metadata
.get("general.architecture")?
.to_string()
.ok()
.cloned()
}
pub fn detect(
metadata: &HashMap<String, gguf_file::Value>,
) -> std::result::Result<Self, String> {
let Some(name) = Self::arch_name(metadata) else {
return Err(
"GGUF metadata has no `general.architecture` key — the file is corrupt \
or is not a model file"
.to_string(),
);
};
if let Some(arch) = Self::from_name(&name) {
return Ok(arch);
}
if Self::is_known_llama_cpp_arch(&name) {
Err(format!(
"Model architecture '{name}' is a known llama.cpp architecture, but no \
pure-Rust quantized loader exists for it in candle yet. \
Joshua currently supports: {}.",
Self::list_known()
))
} else {
Err(format!(
"Unrecognised GGUF architecture '{name}'. \
Joshua currently supports: {}.",
Self::list_known()
))
}
}
pub fn is_known_llama_cpp_arch(name: &str) -> bool {
Self::from_name(name).is_some() || KNOWN_UNSUPPORTED_ARCHS.contains(&name)
}
pub fn display_name(&self) -> &'static str {
match self {
Self::Llama => "Llama (also Mistral, Mixtral, TinyLlama, SmolLM, Yi, …)",
Self::Gemma => "Gemma / Gemma 2 / Gemma 3",
Self::Glm4 => "GLM-4",
Self::Lfm2 => "LFM2",
Self::Phi2 => "Phi-1 / Phi-1.5 / Phi-2",
Self::Phi3 => "Phi-3",
Self::Qwen2 => "Qwen2 / Qwen2.5",
Self::Qwen3 => "Qwen3",
Self::Qwen3Moe => "Qwen3-MoE",
Self::DeepSeek2 => "DeepSeek-V2 / DeepSeek-V3 / Kimi-K2",
Self::DeepSeek4 => "DeepSeek-V4",
}
}
pub fn list_known() -> &'static str {
"llama (incl. Mistral/Mixtral), gemma, gemma2, gemma3, gemma-embedding, \
glm4, lfm2, phi2, phi3, qwen2, qwen3, qwen3moe, deepseek2, deepseek4"
}
}
#[allow(clippy::large_enum_variant)]
pub enum QuantizedModel {
Llama(quantized_llama::ModelWeights),
Gemma(quantized_gemma3::ModelWeights),
Glm4(quantized_glm4::ModelWeights),
Lfm2(quantized_lfm2::ModelWeights),
Phi2(quantized_phi::ModelWeights),
Phi3(quantized_phi3::ModelWeights),
Qwen2(quantized_qwen2::ModelWeights),
Qwen3(quantized_qwen3::ModelWeights),
Qwen3Moe(quantized_qwen3_moe::GGUFQWenMoE),
DeepSeek2(crate::quantized_deepseek2::ModelWeights),
DeepSeek4(crate::quantized_deepseek4::ModelWeights),
}
impl QuantizedModel {
pub fn from_gguf<R: Read + Seek>(
gguf: gguf_file::Content,
reader: &mut R,
device: &Device,
) -> Result<Self> {
Self::from_gguf_mmap(gguf, reader, device, None)
}
pub fn from_gguf_mmap<R: Read + Seek>(
gguf: gguf_file::Content,
reader: &mut R,
device: &Device,
mmap: Option<std::sync::Arc<memmap2::Mmap>>,
) -> Result<Self> {
let arch = Architecture::detect(&gguf.metadata).map_err(candle_core::Error::Msg)?;
tracing::info!("Detected model architecture: {}", arch.display_name());
let raw = if arch == Architecture::DeepSeek4 {
let data_pos = reader.stream_position()?;
let h = (|| -> std::result::Result<_, crate::JoshuaError> {
reader.seek(SeekFrom::Start(0))?;
let h = crate::gguf_ext::read_header(reader)?;
reader.seek(SeekFrom::Start(data_pos))?;
Ok(h)
})()
.map_err(|e| {
let _ = reader.seek(SeekFrom::Start(data_pos));
candle_core::Error::Msg(format!("GGUF header re-read failed: {e}"))
})?;
Some(h)
} else {
None
};
let raw = raw.as_ref();
match arch {
Architecture::Llama => {
quantized_llama::ModelWeights::from_gguf(gguf, reader, device).map(Self::Llama)
}
Architecture::Gemma => {
quantized_gemma3::ModelWeights::from_gguf(gguf, reader, device).map(Self::Gemma)
}
Architecture::Glm4 => {
quantized_glm4::ModelWeights::from_gguf(gguf, reader, device, DType::F32)
.map(Self::Glm4)
}
Architecture::Lfm2 => {
quantized_lfm2::ModelWeights::from_gguf(gguf, reader, device).map(Self::Lfm2)
}
Architecture::Phi2 => {
quantized_phi::ModelWeights::from_gguf(gguf, reader, device).map(Self::Phi2)
}
Architecture::Phi3 => {
quantized_phi3::ModelWeights::from_gguf(false, gguf, reader, device).map(Self::Phi3)
}
Architecture::Qwen2 => {
quantized_qwen2::ModelWeights::from_gguf(gguf, reader, device).map(Self::Qwen2)
}
Architecture::Qwen3 => {
quantized_qwen3::ModelWeights::from_gguf(gguf, reader, device).map(Self::Qwen3)
}
Architecture::Qwen3Moe => {
quantized_qwen3_moe::GGUFQWenMoE::from_gguf(gguf, reader, device, DType::F32)
.map(Self::Qwen3Moe)
}
Architecture::DeepSeek2 => {
crate::quantized_deepseek2::ModelWeights::from_gguf_mmap(gguf, reader, device, mmap)
.map(Self::DeepSeek2)
}
Architecture::DeepSeek4 => crate::quantized_deepseek4::ModelWeights::from_gguf_mmap(
gguf, raw, reader, device, mmap,
)
.map(Self::DeepSeek4),
}
}
pub fn clear_kv_cache(&mut self) -> bool {
match self {
Self::Llama(m) => {
m.clear_kv_cache();
true
}
Self::Qwen2(m) => {
m.clear_kv_cache();
true
}
Self::Qwen3(m) => {
m.clear_kv_cache();
true
}
Self::DeepSeek2(m) => {
m.clear_kv_cache();
true
}
Self::DeepSeek4(m) => {
m.clear_kv_cache();
true
}
_ => false,
}
}
pub fn supports_kv_clear(&self) -> bool {
matches!(
self,
Self::Llama(_)
| Self::Qwen2(_)
| Self::Qwen3(_)
| Self::DeepSeek2(_)
| Self::DeepSeek4(_)
)
}
pub fn forward(&mut self, input: &Tensor, index_pos: usize) -> Result<Tensor> {
match self {
Self::Llama(m) => m.forward(input, index_pos),
Self::Gemma(m) => m.forward(input, index_pos),
Self::Glm4(m) => m.forward(input, index_pos),
Self::Lfm2(m) => m.forward(input, index_pos),
Self::Phi2(m) => m.forward(input, index_pos),
Self::Phi3(m) => m.forward(input, index_pos),
Self::Qwen2(m) => m.forward(input, index_pos),
Self::Qwen3(m) => m.forward(input, index_pos),
Self::Qwen3Moe(m) => m.forward(input, index_pos),
Self::DeepSeek2(m) => m.forward(input, index_pos),
Self::DeepSeek4(m) => m.forward(input, index_pos),
}
}
}
#[cfg(test)]
mod tests {
use super::*;
fn metadata_with_arch(arch: &str) -> HashMap<String, gguf_file::Value> {
let mut m = HashMap::new();
m.insert(
"general.architecture".to_string(),
gguf_file::Value::String(arch.to_string()),
);
m
}
#[test]
fn supported_architectures_resolve() {
for (name, expected) in [
("llama", Architecture::Llama),
("gemma", Architecture::Gemma),
("gemma2", Architecture::Gemma),
("gemma3", Architecture::Gemma),
("gemma-embedding", Architecture::Gemma),
("glm4", Architecture::Glm4),
("lfm2", Architecture::Lfm2),
("phi2", Architecture::Phi2),
("phi3", Architecture::Phi3),
("qwen2", Architecture::Qwen2),
("qwen3", Architecture::Qwen3),
("qwen3moe", Architecture::Qwen3Moe),
("deepseek2", Architecture::DeepSeek2),
("deepseek4", Architecture::DeepSeek4),
] {
assert_eq!(Architecture::from_name(name), Some(expected), "arch {name}");
assert_eq!(
Architecture::from_gguf_metadata(&metadata_with_arch(name)),
Some(expected),
"metadata arch {name}"
);
assert!(Architecture::is_known_llama_cpp_arch(name));
}
}
#[test]
fn known_unsupported_architectures_give_specific_error() {
for name in ["mamba", "gpt2", "deepseek", "rwkv7", "starcoder2"] {
assert_eq!(Architecture::from_name(name), None);
assert!(Architecture::is_known_llama_cpp_arch(name), "arch {name}");
let err = Architecture::detect(&metadata_with_arch(name)).unwrap_err();
assert!(
err.contains("known llama.cpp architecture"),
"error for {name}: {err}"
);
}
}
#[test]
fn unknown_architecture_gives_generic_error() {
assert!(!Architecture::is_known_llama_cpp_arch("not-a-real-arch"));
let err = Architecture::detect(&metadata_with_arch("not-a-real-arch")).unwrap_err();
assert!(err.contains("Unrecognised"), "error: {err}");
}
#[test]
fn missing_architecture_key_gives_corrupt_file_error() {
let err = Architecture::detect(&HashMap::new()).unwrap_err();
assert!(err.contains("general.architecture"), "error: {err}");
}
}