1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
//! **`rope.dimension_sections` ON THE GENERIC PATH** -- what llama.cpp
//! does with M-RoPE sections on a text tower, per architecture, and what
//! that means for a rotation frink decides per architecture.
//!
//! # What it is
//!
//! `llama_hparams::use_mrope()` is `rope_sections[0] > 0 &&
//! rope_sections[1] > 0` (`llama-hparams.cpp:284-286`). The multimodal
//! converters write the sections for the text tower of a vision export
//! (`conversion/glm.py:26-27`), and `llama_model_rope_type` then answers
//! `LLAMA_ROPE_TYPE_MROPE` for the two GLM graphs instead of their text
//! layout (`llama-model.cpp:2698-2701`). With TEXT positions -- one
//! position for every M-RoPE component -- `ggml_rope_multi` in MROPE
//! mode rotates band `i` against band `i + n_dims/2` at angle `pos *
//! freq_i`, which is NEOX rotation band for band. So:
//!
//! - `glm4moe` is NEOX without sections and NEOX with them; libllama's
//! logits on `tests/fixtures/glm4moe_mrope_tiny.gguf` are byte for
//! byte the plain file's (measured, `tests/glm4moe_graphs.rs`). SERVED.
//! - `glm4` is NORM without sections (`:2699`), and the converter
//! PERMUTES a sectioned file's Q/K weights to NEOX order
//! (`glm.py:53-73,78-85`) because the M-RoPE kernel only speaks
//! NEOX; libllama's logits on `tests/fixtures/glm4_mrope_tiny.gguf`
//! differ from the plain file's by 0.72 (measured). A frink rope
//! layout is a property of the architecture (`ArchPath::GenericGqa {
//! rope }`), not of the file, so this file is REFUSED by name.
//!
//! # Reach -- MEASURED
//!
//! `grep -l ROPE_DIMENSION_SECTIONS src/models/*.cpp` over all 155
//! graphs is eleven files. On the generic path: `glm4` and `glm4moe`
//! (above) and `ernie4-5.cpp:5`, which reads the sections into hparams
//! and rotates with `ggml_rope_ext` unconditionally (`llama-model.cpp:
//! 2602` is an unconditional NORM arm), so for ERNIE they are dead
//! metadata and nothing here applies. The other eight are on other
//! engines or refused.
use frink_gguf::{GgufValue, TensorSource};
/// What the generic path does with a file whose sections declare
/// M-RoPE.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum MropeOnText {
/// The architecture's text layout is already NEOX, which is what
/// M-RoPE computes on text positions: served, and the identity is
/// pinned against libllama.
SameAsNeox,
/// The architecture's text layout is NORM and the converter
/// permuted the file to NEOX: refused, with the lines.
RefusedNormBase,
}
/// The two generic-path graphs whose rotation `use_mrope()` switches.
pub const MROPE_READERS: &[(&str, MropeOnText, &str)] = &[
(
"glm4moe",
MropeOnText::SameAsNeox,
"src/models/glm4-moe.cpp:6,145,188; llama-model.cpp:2700",
),
(
"glm4",
MropeOnText::RefusedNormBase,
"src/models/glm4.cpp:5,112-119; llama-model.cpp:2699; conversion/glm.py:53-85",
),
// `qwen35.cpp:5` reads the sections as REQUIRED and `:213-222`
// rotates with `ggml_rope_multi` in IMROPE mode
// (llama-model.cpp:2694-2696) over `rope.dimension_count` bands;
// with one position per component that is NEOX band for band, the
// layout its converter writes (`conversion/qwen.py:378`, partial).
(
"qwen35",
MropeOnText::SameAsNeox,
"src/models/qwen35.cpp:5,213-222; llama-model.cpp:2694-2696",
),
(
"qwen35moe",
MropeOnText::SameAsNeox,
"src/models/qwen35moe.cpp:9; llama-model.cpp:2694-2696",
),
];
/// `llama_hparams::use_mrope()` for a file.
pub fn declares_mrope(file: &impl TensorSource, arch: &str) -> bool {
match file.metadata(&format!("{arch}.rope.dimension_sections")) {
Some(GgufValue::Array(items)) => {
let at = |i: usize| items.get(i).and_then(GgufValue::as_u64).unwrap_or(0);
at(0) > 0 && at(1) > 0
}
_ => false,
}
}
/// The refusal reason for a file that declares M-RoPE on an
/// architecture whose text rotation is not what M-RoPE computes, or
/// `None` when the file may be run.
pub fn mrope_refusal(file: &impl TensorSource, arch: &str) -> Option<String> {
let (_, what, lines) = MROPE_READERS.iter().find(|(n, _, _)| *n == arch)?;
if *what != MropeOnText::RefusedNormBase || !declares_mrope(file, arch) {
return None;
}
Some(format!(
"`{arch}.rope.dimension_sections` declares M-RoPE (a vision export's text tower): \
llama.cpp rotates this file with LLAMA_ROPE_TYPE_MROPE over Q/K weights the converter \
permuted to NEOX order, where the text-only `{arch}` rotates NORM ({lines}). frink \
decides the rotation per architecture, so it stops rather than rotate the wrong pairs \
of every head; libllama's logits for such a file differ from the unpermuted file's by \
0.72 (measured on tests/fixtures/glm4_mrope_tiny.gguf)"
))
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn every_reader_is_a_generic_row_with_the_matching_text_layout() {
for (arch, what, line) in MROPE_READERS {
let profile = crate::capability::resolve_profile(arch)
.unwrap_or_else(|| panic!("`{arch}` ({line}) is not a registered architecture"));
let crate::capability::ArchPath::GenericGqa { rope } = profile.path else {
panic!("`{arch}` is not on the generic path");
};
// The decision follows from the text layout, and the table
// must say the same thing the profile does.
match what {
MropeOnText::SameAsNeox => {
assert_eq!(rope, crate::config::RopeLayout::Neox, "{arch}")
}
MropeOnText::RefusedNormBase => {
assert_eq!(rope, crate::config::RopeLayout::Norm, "{arch}")
}
}
assert!(
crate::capability::AUDITED_GENERIC_GQA.contains(arch),
"{arch}"
);
}
}
}