#![allow(dead_code)]
use anyhow::{anyhow, Result};
use mlx_native::gguf::{GgufFile, MetadataValue};
pub const ARCH_CLIP: &str = "clip";
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum ProjectorType {
Mlp,
Resampler,
Gemma4v,
Qwen3VlMerger,
Other(String),
}
impl ProjectorType {
pub fn from_str_gguf(s: &str) -> Self {
match s {
"mlp" => ProjectorType::Mlp,
"resampler" => ProjectorType::Resampler,
"gemma4v" => ProjectorType::Gemma4v,
"qwen3vl_merger" => ProjectorType::Qwen3VlMerger,
other => ProjectorType::Other(other.to_string()),
}
}
pub fn as_str(&self) -> &str {
match self {
ProjectorType::Mlp => "mlp",
ProjectorType::Resampler => "resampler",
ProjectorType::Gemma4v => "gemma4v",
ProjectorType::Qwen3VlMerger => "qwen3vl_merger",
ProjectorType::Other(s) => s.as_str(),
}
}
pub fn is_supported(&self) -> bool {
matches!(
self,
ProjectorType::Mlp | ProjectorType::Gemma4v | ProjectorType::Qwen3VlMerger
)
}
}
#[derive(Debug, Clone, PartialEq)]
pub struct MmprojConfig {
pub image_size: u32,
pub patch_size: u32,
pub num_patches_side: u32,
pub hidden_size: u32,
pub intermediate_size: u32,
pub num_attention_heads: u32,
pub num_hidden_layers: u32,
pub layer_norm_eps: f32,
pub projector: ProjectorType,
pub image_mean: [f32; 3],
pub image_std: [f32; 3],
pub spatial_merge_size: Option<u32>,
pub projection_dim: Option<u32>,
pub deepstack_indexes: Option<Vec<u32>>,
}
fn read_deepstack_indexes(gguf: &GgufFile, num_hidden_layers: u32) -> Result<Option<Vec<u32>>> {
let raw = match gguf.metadata("clip.vision.is_deepstack_layers") {
Some(v) => v,
None => return Ok(None),
};
let arr = match raw {
MetadataValue::Array(a) => a,
_ => {
return Err(anyhow!(
"mmproj GGUF 'clip.vision.is_deepstack_layers' is not an \
Array(Bool); refusing to silently coerce"
));
}
};
if arr.len() as u32 != num_hidden_layers {
return Err(anyhow!(
"mmproj GGUF 'clip.vision.is_deepstack_layers' length {} \
disagrees with block_count {} — writer mis-encoded the \
per-layer bool array; refusing to silently truncate",
arr.len(),
num_hidden_layers
));
}
let mut out: Vec<u32> = Vec::new();
for (idx, v) in arr.iter().enumerate() {
match v {
MetadataValue::Bool(true) => out.push(idx as u32),
MetadataValue::Bool(false) => {}
_ => {
return Err(anyhow!(
"mmproj GGUF 'clip.vision.is_deepstack_layers'[{}] is \
not a Bool element",
idx
));
}
}
}
Ok(Some(out))
}
impl MmprojConfig {
pub fn from_gguf(gguf: &GgufFile) -> Result<Self> {
let arch = gguf
.metadata_string("general.architecture")
.ok_or_else(|| anyhow!("mmproj GGUF missing general.architecture"))?;
if arch != ARCH_CLIP {
return Err(anyhow!(
"mmproj GGUF architecture is '{}', expected 'clip'",
arch
));
}
let u32_key = |k: &str| -> Result<u32> {
gguf.metadata_u32(k)
.ok_or_else(|| anyhow!("mmproj GGUF missing u32 '{}'", k))
};
let f32_key_default =
|k: &str, default: f32| -> f32 { gguf.metadata_f32(k).unwrap_or(default) };
let image_size = u32_key("clip.vision.image_size")?;
let patch_size = u32_key("clip.vision.patch_size")?;
if patch_size == 0 {
return Err(anyhow!("clip.vision.patch_size = 0"));
}
if image_size % patch_size != 0 {
return Err(anyhow!(
"image_size {} must be divisible by patch_size {}",
image_size,
patch_size
));
}
let num_patches_side = image_size / patch_size;
let hidden_size = u32_key("clip.vision.embedding_length")?;
let intermediate_size = u32_key("clip.vision.feed_forward_length")?;
let num_attention_heads = u32_key("clip.vision.attention.head_count")?;
let num_hidden_layers = u32_key("clip.vision.block_count")?;
let layer_norm_eps = f32_key_default("clip.vision.attention.layer_norm_epsilon", 1e-6);
let projector_str = gguf.metadata_string("clip.projector_type").unwrap_or("mlp");
let projector = ProjectorType::from_str_gguf(projector_str);
let read_triple = |key: &str, default: [f32; 3]| -> [f32; 3] {
match gguf.metadata(key) {
Some(MetadataValue::Array(arr)) if arr.len() == 3 => {
let mut out = [0f32; 3];
for (i, v) in arr.iter().enumerate() {
if let Some(f) = v.as_f32() {
out[i] = f;
} else {
return default;
}
}
out
}
_ => default,
}
};
let image_mean = read_triple("clip.vision.image_mean", [0.5, 0.5, 0.5]);
let image_std = read_triple("clip.vision.image_std", [0.5, 0.5, 0.5]);
let spatial_merge_size = gguf.metadata_u32("clip.vision.spatial_merge_size");
let projection_dim = gguf.metadata_u32("clip.vision.projection_dim");
let deepstack_indexes = read_deepstack_indexes(gguf, num_hidden_layers)?;
Ok(MmprojConfig {
image_size,
patch_size,
num_patches_side,
hidden_size,
intermediate_size,
num_attention_heads,
num_hidden_layers,
layer_norm_eps,
projector,
image_mean,
image_std,
spatial_merge_size,
projection_dim,
deepstack_indexes,
})
}
pub fn num_patches(&self) -> u32 {
self.num_patches_side * self.num_patches_side
}
pub fn preprocess_config(&self) -> super::preprocess::PreprocessConfig {
super::preprocess::PreprocessConfig {
target_size: self.image_size,
mean: self.image_mean,
std: self.image_std,
}
}
}
pub const TENSOR_PATCH_EMBD: &str = "v.patch_embd.weight";
pub const TENSOR_PATCH_EMBD_BIAS: &str = "v.patch_embd.bias";
pub const TENSOR_POS_EMBD: &str = "v.position_embd.weight";
pub const TENSOR_POST_LN_WEIGHT: &str = "v.post_ln.weight";
pub const TENSOR_POST_LN_BIAS: &str = "v.post_ln.bias";
pub const TENSOR_MM_0_WEIGHT: &str = "mm.0.weight";
pub const TENSOR_MM_0_BIAS: &str = "mm.0.bias";
pub const TENSOR_MM_2_WEIGHT: &str = "mm.2.weight";
pub const TENSOR_MM_2_BIAS: &str = "mm.2.bias";
pub const TENSOR_MM_INPUT_PROJECTION_WEIGHT: &str = "mm.input_projection.weight";
pub fn vit_deepstack_tensor(layer_idx: usize, suffix: &str) -> String {
format!("v.deepstack.{}.{}", layer_idx, suffix)
}
pub const DEEPSTACK_REQUIRED_SUFFIXES: &[&str] = &["norm.weight", "fc1.weight", "fc2.weight"];
pub fn vit_layer_tensor(layer_idx: usize, suffix: &str) -> String {
format!("v.blk.{}.{}", layer_idx, suffix)
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ArchProfile {
Gemma4Siglip,
ClipClassic,
Qwen3VlSiglip,
Unknown,
}
impl ArchProfile {
pub fn as_str(&self) -> &'static str {
match self {
ArchProfile::Gemma4Siglip => "gemma4_siglip",
ArchProfile::ClipClassic => "clip_classic",
ArchProfile::Qwen3VlSiglip => "qwen3vl_siglip",
ArchProfile::Unknown => "unknown",
}
}
pub fn is_supported(&self) -> bool {
matches!(
self,
ArchProfile::Gemma4Siglip | ArchProfile::ClipClassic | ArchProfile::Qwen3VlSiglip
)
}
pub fn vision_family(&self) -> VisionFamily {
match self {
ArchProfile::Gemma4Siglip | ArchProfile::ClipClassic => VisionFamily::Gemma,
ArchProfile::Qwen3VlSiglip => VisionFamily::Qwen3Vl,
ArchProfile::Unknown => VisionFamily::Unknown,
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum VisionFamily {
Gemma,
Qwen3Vl,
Unknown,
}
impl VisionFamily {
pub fn placeholder_token_literal(&self) -> Option<&'static str> {
match self {
VisionFamily::Gemma => Some("<|image|>"),
VisionFamily::Qwen3Vl => Some("<|image_pad|>"),
VisionFamily::Unknown => None,
}
}
pub fn marker_pair(&self) -> (&'static str, &'static str) {
match self {
VisionFamily::Gemma => ("", ""),
VisionFamily::Qwen3Vl => ("<|vision_start|>", "<|vision_end|>"),
VisionFamily::Unknown => ("", ""),
}
}
}
pub fn detect_arch_profile(actual_names: &[&str]) -> ArchProfile {
let set: std::collections::HashSet<&str> = actual_names.iter().copied().collect();
if set.contains("v.deepstack.0.fc1.weight") {
return ArchProfile::Qwen3VlSiglip;
}
let has_ln_pair = set.contains("v.blk.0.ln1.weight") && set.contains("v.blk.0.ln2.weight");
let has_gemma4_post_norm = set.contains("v.blk.0.ffn_post_norm.weight")
|| set.contains("v.blk.0.post_ffw_norm.weight");
if has_ln_pair && has_gemma4_post_norm {
return ArchProfile::Gemma4Siglip;
}
if set.contains("v.blk.0.attn_norm.weight") {
return ArchProfile::ClipClassic;
}
ArchProfile::Unknown
}
pub fn detect_arch_profile_with_projector(
projector: &ProjectorType,
actual_names: &[&str],
) -> ArchProfile {
if matches!(projector, ProjectorType::Qwen3VlMerger) {
return ArchProfile::Qwen3VlSiglip;
}
detect_arch_profile(actual_names)
}
pub fn validate_tensor_set(cfg: &MmprojConfig, actual_names: &[&str]) -> Result<()> {
let actual_set: std::collections::HashSet<&str> = actual_names.iter().copied().collect();
let mut missing: Vec<String> = Vec::new();
let required: Vec<String> = vec![TENSOR_PATCH_EMBD.to_string(), TENSOR_POS_EMBD.to_string()];
for layer_idx in 0..cfg.num_hidden_layers as usize {
let attn_out_name = vit_layer_tensor(layer_idx, "attn_out.weight");
if !actual_set.contains(attn_out_name.as_str()) {
missing.push(attn_out_name);
}
let qkv_fused_w = vit_layer_tensor(layer_idx, "attn_qkv.weight");
let q_w = vit_layer_tensor(layer_idx, "attn_q.weight");
let k_w = vit_layer_tensor(layer_idx, "attn_k.weight");
let v_w = vit_layer_tensor(layer_idx, "attn_v.weight");
let has_fused = actual_set.contains(qkv_fused_w.as_str());
let split_present = [&q_w, &k_w, &v_w]
.iter()
.filter(|n| actual_set.contains(n.as_str()))
.count();
let has_full_split = split_present == 3;
match (has_fused, has_full_split, split_present) {
(true, false, 0) => { }
(false, true, _) => { }
(true, true, _) => {
missing.push(format!(
"block {layer_idx}: BOTH fused '{qkv_fused_w}' AND split \
attn_q/k/v are present — producer must emit one form, not both"
));
}
(true, false, _) => {
missing.push(format!(
"block {layer_idx}: fused '{qkv_fused_w}' is present but \
PARTIAL split tensors leaked through (only {split_present}/3 \
of attn_q/k/v.weight present) — refusing to mix conventions"
));
}
(false, false, _) => {
if split_present == 0 {
missing.push(format!(
"block {layer_idx}: missing QKV — neither fused '{qkv_fused_w}' \
nor split attn_q/k/v.weight trio is present"
));
} else {
if !actual_set.contains(q_w.as_str()) {
missing.push(q_w);
}
if !actual_set.contains(k_w.as_str()) {
missing.push(k_w);
}
if !actual_set.contains(v_w.as_str()) {
missing.push(v_w);
}
}
}
}
}
for name in &required {
if !actual_set.contains(name.as_str()) {
missing.push(name.clone());
}
}
if !actual_set.contains(TENSOR_MM_0_WEIGHT)
&& !actual_set.contains(TENSOR_MM_2_WEIGHT)
&& !actual_set.contains(TENSOR_MM_INPUT_PROJECTION_WEIGHT)
{
missing.push(format!(
"{} (or {}, or gemma4v's {})",
TENSOR_MM_0_WEIGHT, TENSOR_MM_2_WEIGHT, TENSOR_MM_INPUT_PROJECTION_WEIGHT,
));
}
if let Some(indexes) = &cfg.deepstack_indexes {
for &flagged_idx in indexes {
if flagged_idx >= cfg.num_hidden_layers {
missing.push(format!(
"deepstack_indexes entry {} exceeds block_count {} \
(writer mis-encoded clip.vision.is_deepstack_layers)",
flagged_idx, cfg.num_hidden_layers
));
continue;
}
for suffix in DEEPSTACK_REQUIRED_SUFFIXES {
let name = vit_deepstack_tensor(flagged_idx as usize, suffix);
if !actual_set.contains(name.as_str()) {
missing.push(name);
}
}
}
}
if !missing.is_empty() {
let total_missing = missing.len();
missing.truncate(10);
let more = if total_missing > 10 {
format!(" (+ {} more)", total_missing - 10)
} else {
String::new()
};
return Err(anyhow!(
"mmproj GGUF is missing {} required tensor(s): {}{}",
total_missing,
missing.join(", "),
more
));
}
if !cfg.projector.is_supported() {
return Err(anyhow!(
"mmproj projector type '{}' is not yet supported by hf2q's ViT \
forward pass. No forward path will succeed for this file.",
cfg.projector.as_str()
));
}
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn projector_type_parse_and_round_trip() {
assert_eq!(ProjectorType::from_str_gguf("mlp"), ProjectorType::Mlp);
assert_eq!(
ProjectorType::from_str_gguf("resampler"),
ProjectorType::Resampler
);
assert_eq!(
ProjectorType::from_str_gguf("gemma4v"),
ProjectorType::Gemma4v
);
assert_eq!(
ProjectorType::from_str_gguf("qwen3vl_merger"),
ProjectorType::Qwen3VlMerger
);
match ProjectorType::from_str_gguf("q-former") {
ProjectorType::Other(s) => assert_eq!(s, "q-former"),
other => panic!("expected Other, got {:?}", other),
}
}
#[test]
fn projector_supported_for_mlp_and_gemma4v() {
assert!(ProjectorType::Mlp.is_supported());
assert!(ProjectorType::Gemma4v.is_supported());
assert!(ProjectorType::Qwen3VlMerger.is_supported());
assert!(!ProjectorType::Resampler.is_supported());
assert!(!ProjectorType::Other("x".into()).is_supported());
}
#[test]
fn projector_as_str_matches_input() {
assert_eq!(ProjectorType::Mlp.as_str(), "mlp");
assert_eq!(ProjectorType::Resampler.as_str(), "resampler");
assert_eq!(ProjectorType::Gemma4v.as_str(), "gemma4v");
assert_eq!(ProjectorType::Qwen3VlMerger.as_str(), "qwen3vl_merger");
assert_eq!(ProjectorType::Other("q-former".into()).as_str(), "q-former");
}
#[test]
fn vit_layer_tensor_formats_blk_prefix() {
assert_eq!(
vit_layer_tensor(0, "attn_q.weight"),
"v.blk.0.attn_q.weight"
);
assert_eq!(
vit_layer_tensor(23, "ffn_down.bias"),
"v.blk.23.ffn_down.bias"
);
}
#[test]
fn tensor_constants_lock_llama_cpp_convention() {
assert_eq!(TENSOR_PATCH_EMBD, "v.patch_embd.weight");
assert_eq!(TENSOR_POS_EMBD, "v.position_embd.weight");
assert_eq!(TENSOR_POST_LN_WEIGHT, "v.post_ln.weight");
assert_eq!(TENSOR_MM_0_WEIGHT, "mm.0.weight");
assert_eq!(TENSOR_MM_2_WEIGHT, "mm.2.weight");
}
#[test]
fn preprocess_config_conversion() {
let mm = MmprojConfig {
image_size: 896,
patch_size: 14,
num_patches_side: 64,
hidden_size: 1152,
intermediate_size: 4304,
num_attention_heads: 16,
num_hidden_layers: 27,
layer_norm_eps: 1e-6,
projector: ProjectorType::Mlp,
image_mean: [0.5, 0.5, 0.5],
image_std: [0.5, 0.5, 0.5],
spatial_merge_size: None,
projection_dim: None,
deepstack_indexes: None,
};
let p = mm.preprocess_config();
assert_eq!(p.target_size, 896);
assert_eq!(p.mean, [0.5, 0.5, 0.5]);
assert_eq!(p.std, [0.5, 0.5, 0.5]);
}
#[test]
fn num_patches_squares_side_count() {
let mm = MmprojConfig {
image_size: 224,
patch_size: 14,
num_patches_side: 16,
hidden_size: 768,
intermediate_size: 3072,
num_attention_heads: 12,
num_hidden_layers: 12,
layer_norm_eps: 1e-6,
projector: ProjectorType::Mlp,
image_mean: [0.5, 0.5, 0.5],
image_std: [0.5, 0.5, 0.5],
spatial_merge_size: None,
projection_dim: None,
deepstack_indexes: None,
};
assert_eq!(mm.num_patches(), 256);
}
fn mlp_cfg(num_layers: u32) -> MmprojConfig {
MmprojConfig {
image_size: 224,
patch_size: 16,
num_patches_side: 14,
hidden_size: 1152,
intermediate_size: 4304,
num_attention_heads: 16,
num_hidden_layers: num_layers,
layer_norm_eps: 1e-6,
projector: ProjectorType::Mlp,
image_mean: [0.5, 0.5, 0.5],
image_std: [0.5, 0.5, 0.5],
spatial_merge_size: None,
projection_dim: None,
deepstack_indexes: None,
}
}
fn minimum_tensor_names(num_layers: u32) -> Vec<String> {
let mut names = vec![
TENSOR_PATCH_EMBD.to_string(),
TENSOR_POS_EMBD.to_string(),
TENSOR_MM_0_WEIGHT.to_string(),
];
for layer_idx in 0..num_layers as usize {
for suffix in [
"attn_q.weight",
"attn_k.weight",
"attn_v.weight",
"attn_out.weight",
] {
names.push(vit_layer_tensor(layer_idx, suffix));
}
}
names
}
#[test]
fn validate_tensor_set_ok_when_minimum_present() {
let cfg = mlp_cfg(2);
let names = minimum_tensor_names(2);
let actual: Vec<&str> = names.iter().map(|s| s.as_str()).collect();
validate_tensor_set(&cfg, &actual).expect("minimum set should pass");
}
#[test]
fn validate_tensor_set_ok_with_mm_2_instead_of_mm_0() {
let cfg = mlp_cfg(1);
let mut names = minimum_tensor_names(1);
names.retain(|n| n != TENSOR_MM_0_WEIGHT);
names.push(TENSOR_MM_2_WEIGHT.to_string());
let actual: Vec<&str> = names.iter().map(|s| s.as_str()).collect();
validate_tensor_set(&cfg, &actual).expect("mm.2 alone should pass");
}
#[test]
fn validate_tensor_set_ok_with_arch_specific_extras() {
let cfg = mlp_cfg(1);
let mut names = minimum_tensor_names(1);
for suffix in [
"ln1.weight",
"ln2.weight",
"post_ffw_norm.weight",
"attn_q_norm.weight",
"attn_k_norm.weight",
"ffn_up.weight",
"ffn_down.weight",
"ffn_gate.weight",
"ffn_norm.weight",
] {
names.push(vit_layer_tensor(0, suffix));
}
names.push("v.std_bias".into());
names.push("v.std_scale".into());
let actual: Vec<&str> = names.iter().map(|s| s.as_str()).collect();
validate_tensor_set(&cfg, &actual).expect("gemma-style extras should pass");
}
#[test]
fn validate_tensor_set_flags_missing_patch_embd() {
let cfg = mlp_cfg(1);
let mut names = minimum_tensor_names(1);
names.retain(|n| n != TENSOR_PATCH_EMBD);
let actual: Vec<&str> = names.iter().map(|s| s.as_str()).collect();
let err = validate_tensor_set(&cfg, &actual).expect_err("should fail");
let msg = format!("{err}");
assert!(msg.contains("v.patch_embd.weight"), "got: {msg}");
}
#[test]
fn validate_tensor_set_flags_missing_projector() {
let cfg = mlp_cfg(1);
let mut names = minimum_tensor_names(1);
names.retain(|n| n != TENSOR_MM_0_WEIGHT);
let actual: Vec<&str> = names.iter().map(|s| s.as_str()).collect();
let err = validate_tensor_set(&cfg, &actual).expect_err("should fail");
let msg = format!("{err}");
assert!(msg.contains("mm.0.weight"), "got: {msg}");
}
#[test]
fn validate_tensor_set_flags_missing_whole_block() {
let cfg = mlp_cfg(2);
let names = minimum_tensor_names(1); let actual: Vec<&str> = names.iter().map(|s| s.as_str()).collect();
let err = validate_tensor_set(&cfg, &actual).expect_err("should fail");
let msg = format!("{err}");
assert!(msg.contains("missing 2 required tensor"), "got: {msg}");
assert!(msg.contains("v.blk.1.attn_out.weight"), "got: {msg}");
assert!(
msg.contains("block 1: missing QKV") && msg.contains("attn_qkv"),
"got: {msg}"
);
}
#[test]
fn validate_tensor_set_rejects_unsupported_projector() {
let mut cfg = mlp_cfg(1);
cfg.projector = ProjectorType::Resampler;
let names = minimum_tensor_names(1);
let actual: Vec<&str> = names.iter().map(|s| s.as_str()).collect();
let err = validate_tensor_set(&cfg, &actual).expect_err("unsupported projector");
let msg = format!("{err}");
assert!(
msg.contains("'resampler' is not yet supported"),
"got: {msg}"
);
}
#[test]
fn validate_tensor_set_rejects_other_projector_with_name_echoed() {
let mut cfg = mlp_cfg(1);
cfg.projector = ProjectorType::Other("q-former".into());
let names = minimum_tensor_names(1);
let actual: Vec<&str> = names.iter().map(|s| s.as_str()).collect();
let err = validate_tensor_set(&cfg, &actual).expect_err("unsupported projector");
let msg = format!("{err}");
assert!(msg.contains("'q-former'"), "got: {msg}");
}
#[test]
fn detect_arch_profile_gemma4_siglip_from_ln1_ln2_post_ffw() {
let names: Vec<&str> = vec![
"v.patch_embd.weight",
"v.blk.0.ln1.weight",
"v.blk.0.ln2.weight",
"v.blk.0.post_ffw_norm.weight",
"v.blk.0.attn_q.weight",
];
assert_eq!(detect_arch_profile(&names), ArchProfile::Gemma4Siglip);
}
#[test]
fn detect_arch_profile_clip_classic_from_attn_norm() {
let names: Vec<&str> = vec![
"v.patch_embd.weight",
"v.blk.0.attn_norm.weight",
"v.blk.0.attn_q.weight",
];
assert_eq!(detect_arch_profile(&names), ArchProfile::ClipClassic);
}
#[test]
fn detect_arch_profile_unknown_when_neither_marker_present() {
let names: Vec<&str> = vec!["v.patch_embd.weight", "v.blk.0.attn_q.weight"];
assert_eq!(detect_arch_profile(&names), ArchProfile::Unknown);
}
#[test]
fn detect_arch_profile_prefers_gemma4_when_both_markers_present() {
let names: Vec<&str> = vec![
"v.patch_embd.weight",
"v.blk.0.ln1.weight",
"v.blk.0.ln2.weight",
"v.blk.0.post_ffw_norm.weight",
"v.blk.0.attn_norm.weight",
];
assert_eq!(detect_arch_profile(&names), ArchProfile::Gemma4Siglip);
}
#[test]
fn detect_arch_profile_with_projector_qwen3vl_no_layer0_deepstack() {
let names: Vec<&str> = vec![
"v.patch_embd.weight",
"v.blk.0.ln1.weight",
"v.blk.0.attn_q.weight",
"v.deepstack.3.fc1.weight",
"v.deepstack.7.fc1.weight",
];
assert_eq!(
detect_arch_profile(&names),
ArchProfile::Unknown,
"tensor-only detector should NOT identify Qwen3-VL when \
v.deepstack.0 marker is absent — this is the precise gap \
the projector-aware variant covers"
);
assert_eq!(
detect_arch_profile_with_projector(&ProjectorType::Qwen3VlMerger, &names),
ArchProfile::Qwen3VlSiglip,
"projector-aware detector must return Qwen3VlSiglip when \
projector_type is Qwen3VlMerger, regardless of which \
deepstack layer indices the file flagged"
);
}
#[test]
fn arch_profile_is_supported_only_for_runtime_paths() {
assert!(ArchProfile::Gemma4Siglip.is_supported());
assert!(ArchProfile::ClipClassic.is_supported());
assert!(ArchProfile::Qwen3VlSiglip.is_supported());
assert!(!ArchProfile::Unknown.is_supported());
}
#[test]
fn arch_profile_as_str_is_snake_case() {
assert_eq!(ArchProfile::Gemma4Siglip.as_str(), "gemma4_siglip");
assert_eq!(ArchProfile::ClipClassic.as_str(), "clip_classic");
assert_eq!(ArchProfile::Qwen3VlSiglip.as_str(), "qwen3vl_siglip");
assert_eq!(ArchProfile::Unknown.as_str(), "unknown");
}
#[test]
fn detect_arch_profile_qwen3vl_from_deepstack_marker() {
let names: Vec<&str> = vec![
"v.patch_embd.weight",
"v.blk.0.ln1.weight",
"v.blk.0.attn_norm.weight",
"v.blk.0.attn_q.weight",
"v.deepstack.0.fc1.weight",
"v.deepstack.0.fc2.weight",
"v.deepstack.0.norm.weight",
];
assert_eq!(detect_arch_profile(&names), ArchProfile::Qwen3VlSiglip);
}
#[test]
fn detect_arch_profile_qwen3vl_marker_wins_over_gemma4_pattern() {
let names: Vec<&str> = vec![
"v.patch_embd.weight",
"v.blk.0.ln1.weight",
"v.blk.0.ln2.weight",
"v.blk.0.ffn_post_norm.weight",
"v.blk.0.attn_q.weight",
"v.deepstack.0.fc1.weight",
];
assert_eq!(detect_arch_profile(&names), ArchProfile::Qwen3VlSiglip);
}
#[test]
fn detect_arch_profile_with_projector_short_circuits_on_qwen3vl_merger() {
let names: Vec<&str> = vec!["v.patch_embd.weight"]; assert_eq!(
detect_arch_profile_with_projector(&ProjectorType::Qwen3VlMerger, &names),
ArchProfile::Qwen3VlSiglip
);
}
#[test]
fn detect_arch_profile_with_projector_falls_through_for_non_qwen3vl() {
let names: Vec<&str> = vec![
"v.patch_embd.weight",
"v.blk.0.ln1.weight",
"v.blk.0.ln2.weight",
"v.blk.0.ffn_post_norm.weight",
];
assert_eq!(
detect_arch_profile_with_projector(&ProjectorType::Gemma4v, &names),
ArchProfile::Gemma4Siglip
);
assert_eq!(
detect_arch_profile_with_projector(&ProjectorType::Mlp, &names),
ArchProfile::Gemma4Siglip
);
}
#[test]
fn vit_deepstack_tensor_formats_deepstack_prefix() {
assert_eq!(
vit_deepstack_tensor(0, "fc1.weight"),
"v.deepstack.0.fc1.weight"
);
assert_eq!(
vit_deepstack_tensor(17, "norm.bias"),
"v.deepstack.17.norm.bias"
);
}
#[test]
fn deepstack_required_suffixes_match_llama_cpp_load_predicate() {
assert_eq!(
DEEPSTACK_REQUIRED_SUFFIXES,
&["norm.weight", "fc1.weight", "fc2.weight"]
);
}
fn qwen3vl_cfg(num_layers: u32, deepstack_indexes: Vec<u32>) -> MmprojConfig {
MmprojConfig {
image_size: 768,
patch_size: 16,
num_patches_side: 48,
hidden_size: 1024,
intermediate_size: 4304,
num_attention_heads: 16,
num_hidden_layers: num_layers,
layer_norm_eps: 1e-6,
projector: ProjectorType::Qwen3VlMerger,
image_mean: [0.5, 0.5, 0.5],
image_std: [0.5, 0.5, 0.5],
spatial_merge_size: Some(2),
projection_dim: Some(2048),
deepstack_indexes: Some(deepstack_indexes),
}
}
fn qwen3vl_minimum_tensor_names(num_layers: u32, flagged: &[u32]) -> Vec<String> {
let mut names = vec![
TENSOR_PATCH_EMBD.to_string(),
TENSOR_POS_EMBD.to_string(),
TENSOR_MM_0_WEIGHT.to_string(),
];
for layer_idx in 0..num_layers as usize {
for suffix in [
"attn_q.weight",
"attn_k.weight",
"attn_v.weight",
"attn_out.weight",
] {
names.push(vit_layer_tensor(layer_idx, suffix));
}
}
for &idx in flagged {
for suffix in DEEPSTACK_REQUIRED_SUFFIXES {
names.push(vit_deepstack_tensor(idx as usize, suffix));
}
}
names
}
#[test]
fn validate_qwen3vl_complete_set_passes_after_wedge_4c5() {
let cfg = qwen3vl_cfg(24, vec![5, 11, 17]);
let names = qwen3vl_minimum_tensor_names(24, &[5, 11, 17]);
let actual: Vec<&str> = names.iter().map(|s| s.as_str()).collect();
validate_tensor_set(&cfg, &actual).expect(
"Wedge-4c.5: tensor-complete Qwen3-VL with split attn_q/k/v must validate cleanly \
now that ProjectorType::Qwen3VlMerger.is_supported() returns true",
);
}
#[test]
fn validate_qwen3vl_flags_missing_deepstack_trio() {
let cfg = qwen3vl_cfg(24, vec![5, 11, 17]);
let names = qwen3vl_minimum_tensor_names(24, &[5, 17]);
let actual: Vec<&str> = names.iter().map(|s| s.as_str()).collect();
let err = validate_tensor_set(&cfg, &actual)
.expect_err("missing deepstack trio at layer 11 must fail");
let msg = format!("{err}");
assert!(msg.contains("v.deepstack.11.norm.weight"), "got: {msg}");
assert!(msg.contains("v.deepstack.11.fc1.weight"), "got: {msg}");
assert!(msg.contains("v.deepstack.11.fc2.weight"), "got: {msg}");
}
#[test]
fn validate_qwen3vl_flags_out_of_range_deepstack_index() {
let cfg = qwen3vl_cfg(4, vec![10]); let names = qwen3vl_minimum_tensor_names(4, &[]);
let actual: Vec<&str> = names.iter().map(|s| s.as_str()).collect();
let err = validate_tensor_set(&cfg, &actual)
.expect_err("deepstack index >= block_count must fail");
let msg = format!("{err}");
assert!(
msg.contains("deepstack_indexes entry 10 exceeds block_count 4"),
"got: {msg}"
);
}
#[test]
fn validate_qwen3vl_no_deepstack_indexes_is_lenient() {
let mut cfg = qwen3vl_cfg(4, vec![]);
cfg.deepstack_indexes = None;
cfg.projector = ProjectorType::Mlp;
let names = qwen3vl_minimum_tensor_names(4, &[]);
let actual: Vec<&str> = names.iter().map(|s| s.as_str()).collect();
validate_tensor_set(&cfg, &actual)
.expect("None deepstack_indexes + complete universal set must pass");
}
#[test]
fn validate_qwen3vl_empty_deepstack_indexes_requires_no_extras() {
let mut cfg = qwen3vl_cfg(4, vec![]);
cfg.projector = ProjectorType::Mlp; let names = qwen3vl_minimum_tensor_names(4, &[]);
let actual: Vec<&str> = names.iter().map(|s| s.as_str()).collect();
validate_tensor_set(&cfg, &actual)
.expect("empty deepstack_indexes with complete universal set must pass");
}
fn qwen3vl_fused_qkv_tensor_names(num_layers: u32, flagged: &[u32]) -> Vec<String> {
let mut names = vec![
TENSOR_PATCH_EMBD.to_string(),
TENSOR_POS_EMBD.to_string(),
TENSOR_MM_0_WEIGHT.to_string(),
];
for layer_idx in 0..num_layers as usize {
names.push(vit_layer_tensor(layer_idx, "attn_qkv.weight"));
names.push(vit_layer_tensor(layer_idx, "attn_out.weight"));
}
for &idx in flagged {
for suffix in DEEPSTACK_REQUIRED_SUFFIXES {
names.push(vit_deepstack_tensor(idx as usize, suffix));
}
}
names
}
#[test]
fn validate_qwen3vl_fused_attn_qkv_accepted() {
let cfg = qwen3vl_cfg(24, vec![5, 11, 17]);
let names = qwen3vl_fused_qkv_tensor_names(24, &[5, 11, 17]);
let actual: Vec<&str> = names.iter().map(|s| s.as_str()).collect();
validate_tensor_set(&cfg, &actual).expect(
"Wedge-4c.5: tensor-complete Qwen3-VL with FUSED attn_qkv must validate cleanly",
);
}
#[test]
fn validate_qwen3vl_split_attn_qkv_still_accepted() {
let cfg = qwen3vl_cfg(24, vec![5, 11, 17]);
let names = qwen3vl_minimum_tensor_names(24, &[5, 11, 17]);
let actual: Vec<&str> = names.iter().map(|s| s.as_str()).collect();
validate_tensor_set(&cfg, &actual)
.expect("split attn_q/k/v.weight trio must remain a valid Qwen3-VL form");
}
#[test]
fn validate_qwen3vl_mixed_qkv_form_rejected() {
let cfg = qwen3vl_cfg(4, vec![]);
let mut names = qwen3vl_minimum_tensor_names(4, &[]);
names.push(vit_layer_tensor(0, "attn_qkv.weight"));
let actual: Vec<&str> = names.iter().map(|s| s.as_str()).collect();
let err = validate_tensor_set(&cfg, &actual)
.expect_err("BOTH fused AND split present at the same block must fail");
let msg = format!("{err}");
assert!(
msg.contains("BOTH fused") && msg.contains("attn_qkv"),
"error must call out the both-forms case at block 0; got: {msg}"
);
}
#[test]
fn validate_qwen3vl_missing_qkv_names_fused_alternative() {
let cfg = qwen3vl_cfg(2, vec![]);
let mut names = vec![
TENSOR_PATCH_EMBD.to_string(),
TENSOR_POS_EMBD.to_string(),
TENSOR_MM_0_WEIGHT.to_string(),
];
for layer_idx in 0..2usize {
names.push(vit_layer_tensor(layer_idx, "attn_out.weight"));
}
let actual: Vec<&str> = names.iter().map(|s| s.as_str()).collect();
let err =
validate_tensor_set(&cfg, &actual).expect_err("missing QKV in any form must fail");
let msg = format!("{err}");
assert!(
msg.contains("attn_qkv") && msg.contains("attn_q/k/v"),
"error must name BOTH the fused alternative ('attn_qkv') AND \
the split form ('attn_q/k/v') so the producer can pick; got: {msg}"
);
}
}