use std::io::{self, Read};
const MAGIC: u32 = 0x4655_4747;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum ValueType {
U8,
I8,
U16,
I16,
U32,
I32,
F32,
Bool,
String,
Array,
U64,
I64,
F64,
}
impl ValueType {
fn from_tag(tag: u32) -> Result<ValueType, String> {
Ok(match tag {
0 => ValueType::U8,
1 => ValueType::I8,
2 => ValueType::U16,
3 => ValueType::I16,
4 => ValueType::U32,
5 => ValueType::I32,
6 => ValueType::F32,
7 => ValueType::Bool,
8 => ValueType::String,
9 => ValueType::Array,
10 => ValueType::U64,
11 => ValueType::I64,
12 => ValueType::F64,
other => return Err(format!("unknown GGUF value type tag {other}")),
})
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct KvShape {
pub block_count: u64,
pub head_count_kv: u64,
pub key_length: u64,
pub value_length: u64,
pub context_length: u64,
}
impl KvShape {
pub fn kv_bytes_for(&self, total_ctx_tokens: u64) -> u64 {
self.block_count
.saturating_mul(self.head_count_kv)
.saturating_mul(self.key_length.saturating_add(self.value_length))
.saturating_mul(total_ctx_tokens)
.saturating_mul(2)
}
}
#[derive(Default)]
struct RawFacts {
block_count: Option<u64>,
head_count: Option<u64>,
head_count_kv: Option<u64>,
key_length: Option<u64>,
value_length: Option<u64>,
embedding_length: Option<u64>,
context_length: Option<u64>,
}
pub fn read_kv_shape<R: Read>(r: &mut R) -> Result<KvShape, String> {
let magic = read_u32(r).map_err(|e| format!("reading GGUF magic: {e}"))?;
if magic != MAGIC {
return Err(format!(
"not a GGUF file: magic was {magic:#010x}, expected {MAGIC:#010x}"
));
}
let _version = read_u32(r).map_err(|e| format!("reading GGUF version: {e}"))?;
let _tensor_count = read_u64(r).map_err(|e| format!("reading tensor count: {e}"))?;
let kv_count = read_u64(r).map_err(|e| format!("reading metadata count: {e}"))?;
let mut facts = RawFacts::default();
for i in 0..kv_count {
let key = read_string(r).map_err(|e| format!("metadata entry {i} key: {e}"))?;
let tag = read_u32(r).map_err(|e| format!("metadata entry {i} type: {e}"))?;
let vtype = ValueType::from_tag(tag).map_err(|e| format!("entry {i} ({key}): {e}"))?;
let wants_value = key.ends_with(".block_count")
|| key.ends_with(".attention.head_count_kv")
|| key.ends_with(".attention.head_count")
|| key.ends_with(".attention.key_length")
|| key.ends_with(".attention.value_length")
|| key.ends_with(".embedding_length")
|| key.ends_with(".context_length");
if !wants_value || vtype == ValueType::Array || vtype == ValueType::String {
skip_value(r, vtype).map_err(|e| format!("skipping entry {i} ({key}): {e}"))?;
continue;
}
let value = read_scalar_as_u64(r, vtype)
.map_err(|e| format!("reading entry {i} ({key}) as an integer: {e}"))?;
if key.ends_with(".block_count") {
facts.block_count = Some(value);
} else if key.ends_with(".attention.head_count_kv") {
facts.head_count_kv = Some(value);
} else if key.ends_with(".attention.head_count") {
facts.head_count = Some(value);
} else if key.ends_with(".attention.key_length") {
facts.key_length = Some(value);
} else if key.ends_with(".attention.value_length") {
facts.value_length = Some(value);
} else if key.ends_with(".embedding_length") {
facts.embedding_length = Some(value);
} else if key.ends_with(".context_length") {
facts.context_length = Some(value);
}
}
build_shape(facts)
}
fn build_shape(facts: RawFacts) -> Result<KvShape, String> {
let block_count = facts.block_count.ok_or("missing \"*.block_count\"")?;
let context_length = facts.context_length.ok_or("missing \"*.context_length\"")?;
let head_count_kv = facts
.head_count_kv
.or(facts.head_count)
.ok_or("missing both \"*.attention.head_count_kv\" and \"*.attention.head_count\"")?;
let derived_head_dim = match (facts.embedding_length, facts.head_count) {
(Some(embd), Some(heads)) if heads > 0 => Some(embd / heads),
_ => None,
};
let key_length = facts
.key_length
.or(facts.value_length)
.or(derived_head_dim)
.ok_or(
"missing \"*.attention.key_length\" and nothing to derive it from \
(\"*.embedding_length\" and \"*.attention.head_count\")",
)?;
let value_length = facts
.value_length
.or(facts.key_length)
.unwrap_or(key_length);
Ok(KvShape {
block_count,
head_count_kv,
key_length,
value_length,
context_length,
})
}
fn read_exact<R: Read>(r: &mut R, n: usize) -> io::Result<Vec<u8>> {
let mut buf = vec![0u8; n];
r.read_exact(&mut buf)?;
Ok(buf)
}
fn read_u32<R: Read>(r: &mut R) -> io::Result<u32> {
let b = read_exact(r, 4)?;
Ok(u32::from_le_bytes([b[0], b[1], b[2], b[3]]))
}
fn read_u64<R: Read>(r: &mut R) -> io::Result<u64> {
let b = read_exact(r, 8)?;
Ok(u64::from_le_bytes([
b[0], b[1], b[2], b[3], b[4], b[5], b[6], b[7],
]))
}
fn read_string<R: Read>(r: &mut R) -> io::Result<String> {
let len = read_u64(r)? as usize;
let bytes = read_exact(r, len)?;
Ok(String::from_utf8_lossy(&bytes).into_owned())
}
fn fixed_width(vtype: ValueType) -> Option<usize> {
Some(match vtype {
ValueType::U8 | ValueType::I8 | ValueType::Bool => 1,
ValueType::U16 | ValueType::I16 => 2,
ValueType::U32 | ValueType::I32 | ValueType::F32 => 4,
ValueType::U64 | ValueType::I64 | ValueType::F64 => 8,
ValueType::String | ValueType::Array => return None,
})
}
fn skip_value<R: Read>(r: &mut R, vtype: ValueType) -> io::Result<()> {
match vtype {
ValueType::String => {
read_string(r)?;
}
ValueType::Array => {
let elem_tag = read_u32(r)?;
let elem_type = ValueType::from_tag(elem_tag)
.map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?;
let len = read_u64(r)?;
for _ in 0..len {
skip_value(r, elem_type)?;
}
}
other => {
let width = fixed_width(other).expect("fixed_width covers every non-string/array tag");
read_exact(r, width)?;
}
}
Ok(())
}
fn read_scalar_as_u64<R: Read>(r: &mut R, vtype: ValueType) -> io::Result<u64> {
Ok(match vtype {
ValueType::U8 | ValueType::I8 | ValueType::Bool => read_exact(r, 1)?[0] as u64,
ValueType::U16 | ValueType::I16 => {
let b = read_exact(r, 2)?;
u16::from_le_bytes([b[0], b[1]]) as u64
}
ValueType::U32 | ValueType::I32 => {
let b = read_exact(r, 4)?;
u32::from_le_bytes([b[0], b[1], b[2], b[3]]) as u64
}
ValueType::F32 => {
let b = read_exact(r, 4)?;
f32::from_le_bytes([b[0], b[1], b[2], b[3]]) as u64
}
ValueType::U64 | ValueType::I64 => read_u64(r)?,
ValueType::F64 => {
let b = read_exact(r, 8)?;
f64::from_le_bytes([b[0], b[1], b[2], b[3], b[4], b[5], b[6], b[7]]) as u64
}
ValueType::String | ValueType::Array => {
return Err(io::Error::new(
io::ErrorKind::InvalidData,
"expected a numeric value",
))
}
})
}
#[cfg(test)]
mod tests {
use super::*;
fn qwen_style_bytes() -> Vec<u8> {
let mut b = Vec::new();
b.extend_from_slice(&MAGIC.to_le_bytes());
b.extend_from_slice(&3u32.to_le_bytes()); b.extend_from_slice(&0u64.to_le_bytes()); let entries: &[(&str, ValueType, u64)] = &[
("qwen2.block_count", ValueType::U32, 48),
("qwen2.attention.head_count", ValueType::U32, 40),
("qwen2.attention.head_count_kv", ValueType::U32, 8),
("qwen2.attention.key_length", ValueType::U32, 128),
("qwen2.attention.value_length", ValueType::U32, 128),
("qwen2.context_length", ValueType::U32, 32768),
("qwen2.embedding_length", ValueType::U32, 5120),
];
b.extend_from_slice(&(entries.len() as u64).to_le_bytes());
for (key, vtype, value) in entries {
write_string(&mut b, key);
write_scalar(&mut b, *vtype, *value);
}
b
}
fn write_string(b: &mut Vec<u8>, s: &str) {
b.extend_from_slice(&(s.len() as u64).to_le_bytes());
b.extend_from_slice(s.as_bytes());
}
fn write_scalar(b: &mut Vec<u8>, vtype: ValueType, value: u64) {
let tag: u32 = match vtype {
ValueType::U8 => 0,
ValueType::U16 => 2,
ValueType::U32 => 4,
ValueType::U64 => 10,
_ => panic!("test helper only covers unsigned scalars"),
};
b.extend_from_slice(&tag.to_le_bytes());
match vtype {
ValueType::U8 => b.push(value as u8),
ValueType::U16 => b.extend_from_slice(&(value as u16).to_le_bytes()),
ValueType::U32 => b.extend_from_slice(&(value as u32).to_le_bytes()),
ValueType::U64 => b.extend_from_slice(&value.to_le_bytes()),
_ => unreachable!(),
}
}
#[test]
fn reads_the_shape_a_gqa_model_actually_declares() {
let bytes = qwen_style_bytes();
let shape = read_kv_shape(&mut &bytes[..]).unwrap();
assert_eq!(shape.block_count, 48);
assert_eq!(shape.head_count_kv, 8, "GQA: kv heads, not query heads");
assert_eq!(shape.key_length, 128);
assert_eq!(shape.value_length, 128);
assert_eq!(shape.context_length, 32768);
}
#[test]
fn a_file_with_no_gqa_metadata_falls_back_to_head_count() {
let mut b = Vec::new();
b.extend_from_slice(&MAGIC.to_le_bytes());
b.extend_from_slice(&3u32.to_le_bytes());
b.extend_from_slice(&0u64.to_le_bytes());
let entries: &[(&str, u64)] = &[
("llama.block_count", 32),
("llama.attention.head_count", 32),
("llama.attention.key_length", 128),
("llama.context_length", 8192),
];
b.extend_from_slice(&(entries.len() as u64).to_le_bytes());
for (key, value) in entries {
write_string(&mut b, key);
write_scalar(&mut b, ValueType::U32, *value);
}
let shape = read_kv_shape(&mut &b[..]).unwrap();
assert_eq!(shape.head_count_kv, 32, "falls back to head_count");
assert_eq!(shape.value_length, 128, "falls back to key_length");
}
#[test]
fn key_and_value_length_derive_from_embedding_and_head_count_when_absent() {
let mut b = Vec::new();
b.extend_from_slice(&MAGIC.to_le_bytes());
b.extend_from_slice(&3u32.to_le_bytes());
b.extend_from_slice(&0u64.to_le_bytes());
let entries: &[(&str, u64)] = &[
("gptneox.block_count", 24),
("gptneox.attention.head_count", 16),
("gptneox.embedding_length", 2048), ("gptneox.context_length", 2048),
];
b.extend_from_slice(&(entries.len() as u64).to_le_bytes());
for (key, value) in entries {
write_string(&mut b, key);
write_scalar(&mut b, ValueType::U32, *value);
}
let shape = read_kv_shape(&mut &b[..]).unwrap();
assert_eq!(shape.key_length, 128);
assert_eq!(shape.value_length, 128);
}
#[test]
fn a_vocabulary_sized_array_is_skipped_not_allocated() {
let mut b = Vec::new();
b.extend_from_slice(&MAGIC.to_le_bytes());
b.extend_from_slice(&3u32.to_le_bytes());
b.extend_from_slice(&0u64.to_le_bytes());
b.extend_from_slice(&5u64.to_le_bytes());
write_string(&mut b, "qwen2.block_count");
write_scalar(&mut b, ValueType::U32, 48);
write_string(&mut b, "tokenizer.ggml.tokens");
b.extend_from_slice(&9u32.to_le_bytes()); b.extend_from_slice(&8u32.to_le_bytes()); b.extend_from_slice(&2000u64.to_le_bytes()); for i in 0..2000u32 {
write_string(&mut b, &format!("tok{i}"));
}
write_string(&mut b, "qwen2.attention.head_count_kv");
write_scalar(&mut b, ValueType::U32, 8);
write_string(&mut b, "qwen2.attention.key_length");
write_scalar(&mut b, ValueType::U32, 128);
write_string(&mut b, "qwen2.context_length");
write_scalar(&mut b, ValueType::U32, 32768);
let shape = read_kv_shape(&mut &b[..]).unwrap();
assert_eq!(shape.block_count, 48);
assert_eq!(shape.head_count_kv, 8);
assert_eq!(shape.context_length, 32768);
}
#[test]
fn the_wrong_magic_is_refused_by_name() {
let bytes = [0u8; 16];
let err = read_kv_shape(&mut &bytes[..]).unwrap_err();
assert!(err.contains("not a GGUF file"), "got: {err}");
}
#[test]
fn a_truncated_file_is_a_named_error_not_a_panic() {
let bytes = &MAGIC.to_le_bytes()[..2]; let err = read_kv_shape(&mut &bytes[..]).unwrap_err();
assert!(!err.is_empty());
}
#[test]
fn missing_block_count_is_a_named_error() {
let mut b = Vec::new();
b.extend_from_slice(&MAGIC.to_le_bytes());
b.extend_from_slice(&3u32.to_le_bytes());
b.extend_from_slice(&0u64.to_le_bytes());
b.extend_from_slice(&0u64.to_le_bytes()); let err = read_kv_shape(&mut &b[..]).unwrap_err();
assert!(err.contains("block_count"), "got: {err}");
}
#[test]
fn matches_the_shape_measured_on_the_real_muse_incident() {
let shape = KvShape {
block_count: 32,
head_count_kv: 8, key_length: 128,
value_length: 128,
context_length: 8192,
};
let total_ctx = 32_768; let bytes = shape.kv_bytes_for(total_ctx);
let gib = bytes as f64 / (1024.0 * 1024.0 * 1024.0);
assert!(
(3.9..4.1).contains(&gib),
"expected ~4 GiB for this one model's KV cache, got {gib:.1} GiB"
);
}
#[test]
fn doubling_context_doubles_kv_bytes() {
let shape = KvShape {
block_count: 32,
head_count_kv: 8,
key_length: 128,
value_length: 128,
context_length: 8192,
};
let one = shape.kv_bytes_for(4096);
let two = shape.kv_bytes_for(8192);
assert_eq!(two, one * 2);
}
#[test]
fn zero_context_is_zero_bytes_not_a_panic() {
let shape = KvShape {
block_count: 32,
head_count_kv: 8,
key_length: 128,
value_length: 128,
context_length: 8192,
};
assert_eq!(shape.kv_bytes_for(0), 0);
}
}