use std::collections::BTreeMap;
use std::fs;
use std::path::PathBuf;
use crate::format::{
ArchInfo, CommonTensor, ExpertsLayout, FileEntry, LAYOUT_FILE, LayerLayout, Manifest,
Projection, ProjectionName, QuantInfo, RVMP_VERSION, SourceInfo, layer_file_name, sha256_file,
testutil::TempDir, write_layout, write_manifest,
};
use crate::kernels::quants::{QuantFormat, f32_to_f16};
pub(crate) const N_LAYERS: u32 = 2;
pub(crate) const N_EXPERTS: u32 = 4;
pub(crate) const HIDDEN: usize = 256;
pub(crate) const MOE_INTERMEDIATE: usize = 256;
pub(crate) const N_HEADS: usize = 4;
pub(crate) const N_KV_HEADS: usize = 2;
pub(crate) const HEAD_DIM: usize = 64;
pub(crate) const VOCAB: usize = 32;
pub(crate) struct Geometry {
pub(crate) n_layers: u32,
pub(crate) n_experts: u32,
pub(crate) top_k: u32,
pub(crate) hidden: usize,
pub(crate) moe_intermediate: usize,
pub(crate) n_heads: usize,
pub(crate) n_kv_heads: usize,
pub(crate) head_dim: usize,
pub(crate) vocab: usize,
pub(crate) model_id: &'static str,
pub(crate) hf_repo: &'static str,
pub(crate) source_file: &'static str,
pub(crate) norm_value: fn(&str, usize) -> f32,
pub(crate) router_value: fn(&str, usize) -> f32,
}
impl Geometry {
fn q_dim(&self) -> usize {
self.n_heads * self.head_dim
}
fn kv_dim(&self) -> usize {
self.n_kv_heads * self.head_dim
}
pub(crate) fn arch(&self) -> ArchInfo {
ArchInfo {
n_layers: self.n_layers,
n_experts: self.n_experts,
top_k: self.top_k,
hidden: self.hidden as u32,
moe_intermediate: self.moe_intermediate as u32,
n_heads: self.n_heads as u32,
n_kv_heads: self.n_kv_heads as u32,
head_dim: self.head_dim as u32,
vocab: self.vocab as u32,
context_length: 4096,
rope_theta: 1e7,
rms_eps: 1e-6,
norm_topk_prob: true,
tie_embeddings: false,
shared_expert: false,
sliding_window: None,
}
}
}
pub(crate) const NARROW: Geometry = Geometry {
n_layers: N_LAYERS,
n_experts: N_EXPERTS,
top_k: 2,
hidden: HIDDEN,
moe_intermediate: MOE_INTERMEDIATE,
n_heads: N_HEADS,
n_kv_heads: N_KV_HEADS,
head_dim: HEAD_DIM,
vocab: VOCAB,
model_id: "fixture-moe-2l",
hf_repo: "test/fixture",
source_file: "fixture-Q4_K_M.gguf",
norm_value: f32_pattern,
router_value: f32_pattern,
};
pub(crate) struct Fixture {
_tmp: TempDir,
pub(crate) root: PathBuf,
pub(crate) manifest: Manifest,
pub(crate) layout: ExpertsLayout,
}
pub(crate) fn f32_pattern(name: &str, index: usize) -> f32 {
(index as f32) * 0.25 - (name.len() as f32) * 0.5
}
fn pattern_byte(seed: usize, block: usize, byte: usize) -> u8 {
(seed.wrapping_mul(31) ^ block.wrapping_mul(7) ^ byte.wrapping_mul(3)) as u8
}
fn quant_bytes(format: QuantFormat, rows: usize, in_dim: usize, seed: usize) -> Vec<u8> {
let row_bytes = format.row_bytes(in_dim).expect("fixture dims divide");
let block_bytes = format.block_bytes();
let n_blocks = rows * row_bytes / block_bytes;
let mut out = vec![0u8; rows * row_bytes];
for block in 0..n_blocks {
let base = block * block_bytes;
for byte in 0..block_bytes {
out[base + byte] = pattern_byte(seed, block, byte);
}
let d = f32_to_f16(0.5).to_le_bytes();
let dmin = f32_to_f16(0.25).to_le_bytes();
match format {
QuantFormat::Q4_K | QuantFormat::Q5_K => {
out[base..base + 2].copy_from_slice(&d);
out[base + 2..base + 4].copy_from_slice(&dmin);
}
QuantFormat::Q6_K => out[base + 208..base + 210].copy_from_slice(&d),
QuantFormat::Q8_0 => out[base..base + 2].copy_from_slice(&d),
QuantFormat::Q8_K => unreachable!("activation-only format"),
}
}
out
}
fn f32_bytes(name: &str, elements: usize, payload: fn(&str, usize) -> f32) -> Vec<u8> {
let mut out = Vec::with_capacity(elements * 4);
for i in 0..elements {
out.extend_from_slice(&payload(name, i).to_le_bytes());
}
out
}
pub(crate) fn fixture_arch() -> ArchInfo {
NARROW.arch()
}
struct TensorDef {
dtype: &'static str,
bytes: Vec<u8>,
}
fn common_defs(g: &Geometry) -> BTreeMap<String, TensorDef> {
let mut defs = BTreeMap::new();
let mut quant = |name: String, format: QuantFormat, rows: usize, in_dim: usize| {
let dtype = match format {
QuantFormat::Q4_K => "q4_k",
QuantFormat::Q5_K => "q5_k",
QuantFormat::Q6_K => "q6_k",
QuantFormat::Q8_0 => "q8_0",
QuantFormat::Q8_K => unreachable!("activation-only format"),
};
let bytes = quant_bytes(format, rows, in_dim, name.len() + rows);
defs.insert(name, TensorDef { dtype, bytes });
};
quant(
"token_embd.weight".to_owned(),
QuantFormat::Q4_K,
g.vocab,
g.hidden,
);
quant(
"output.weight".to_owned(),
QuantFormat::Q6_K,
g.vocab,
g.hidden,
);
let q_dim = g.q_dim();
let kv_dim = g.kv_dim();
for layer in 0..g.n_layers {
quant(
format!("blk.{layer}.attn_q.weight"),
QuantFormat::Q4_K,
q_dim,
g.hidden,
);
quant(
format!("blk.{layer}.attn_k.weight"),
QuantFormat::Q8_0,
kv_dim,
g.hidden,
);
let v_format = if layer == 0 {
QuantFormat::Q6_K
} else {
QuantFormat::Q4_K
};
quant(
format!("blk.{layer}.attn_v.weight"),
v_format,
kv_dim,
g.hidden,
);
quant(
format!("blk.{layer}.attn_output.weight"),
QuantFormat::Q5_K,
g.hidden,
q_dim,
);
}
let mut f32_def = |name: String, elements: usize, payload: fn(&str, usize) -> f32| {
let bytes = f32_bytes(&name, elements, payload);
defs.insert(
name,
TensorDef {
dtype: "f32",
bytes,
},
);
};
f32_def("output_norm.weight".to_owned(), g.hidden, g.norm_value);
for layer in 0..g.n_layers {
for (stem, elements) in [
("attn_norm", g.hidden),
("ffn_norm", g.hidden),
("attn_q_norm", g.head_dim),
("attn_k_norm", g.head_dim),
] {
f32_def(format!("blk.{layer}.{stem}.weight"), elements, g.norm_value);
}
f32_def(
format!("blk.{layer}.ffn_gate_inp.weight"),
g.n_experts as usize * g.hidden,
g.router_value,
);
}
defs
}
pub(crate) fn build_install(tag: &str) -> Fixture {
build_install_with(tag, &NARROW)
}
pub(crate) fn build_install_with(tag: &str, g: &Geometry) -> Fixture {
let tmp = TempDir::new(tag);
let root = tmp.path().join("model.rvmp");
fs::create_dir_all(root.join("experts")).expect("create install dirs");
let defs = common_defs(g);
let mut common_tensors = BTreeMap::new();
let mut common = Vec::new();
for (name, def) in &defs {
let offset = (common.len() as u64).next_multiple_of(64);
common.resize(offset as usize, 0);
common.extend_from_slice(&def.bytes);
common_tensors.insert(
name.clone(),
CommonTensor {
offset,
len: def.bytes.len() as u64,
dtype: def.dtype.to_owned(),
},
);
}
fs::write(root.join("common.bin"), &common).expect("write common.bin");
let mut layers = Vec::new();
for layer in 0..g.n_layers {
let down_format = if layer == 0 {
QuantFormat::Q6_K
} else {
QuantFormat::Q4_K
};
let slabs: [(ProjectionName, QuantFormat, usize, usize); 3] = [
(
ProjectionName::Gate,
QuantFormat::Q4_K,
g.moe_intermediate,
g.hidden,
),
(
ProjectionName::Up,
QuantFormat::Q4_K,
g.moe_intermediate,
g.hidden,
),
(
ProjectionName::Down,
down_format,
g.hidden,
g.moe_intermediate,
),
];
let mut projections = Vec::new();
let mut cursor: u64 = 0;
for (name, format, rows, in_dim) in slabs {
let offset_in_blob = cursor.next_multiple_of(4096);
let len = (format.row_bytes(in_dim).unwrap() * rows) as u64;
projections.push(Projection {
name,
offset_in_blob,
len,
quant: match format {
QuantFormat::Q4_K => "q4_k",
QuantFormat::Q6_K => "q6_k",
_ => unreachable!("fixture expert formats"),
}
.to_owned(),
});
cursor = offset_in_blob + len;
}
let stride = cursor.next_multiple_of(4096);
let mut file_bytes = vec![0u8; (stride * u64::from(g.n_experts)) as usize];
for expert in 0..g.n_experts {
let blob_base = (u64::from(expert) * stride) as usize;
for (p, ((_, format, rows, in_dim), projection)) in
slabs.iter().zip(&projections).enumerate()
{
let seed = layer as usize * 1009 + expert as usize * 101 + p * 13;
let bytes = quant_bytes(*format, *rows, *in_dim, seed);
let start = blob_base + projection.offset_in_blob as usize;
file_bytes[start..start + bytes.len()].copy_from_slice(&bytes);
}
}
let file = layer_file_name(layer);
fs::write(root.join(&file), &file_bytes).expect("write layer file");
layers.push(LayerLayout {
file,
stride,
n_experts: g.n_experts,
projections,
});
}
let layout = ExpertsLayout { layers };
write_layout(&root, &layout).expect("write layout");
let mut files = BTreeMap::new();
let mut record = |name: String| {
let path = root.join(&name);
files.insert(
name,
FileEntry {
size: fs::metadata(&path).unwrap().len(),
sha256: sha256_file(&path).unwrap(),
},
);
};
record("common.bin".to_owned());
record(LAYOUT_FILE.to_owned());
for layer in 0..g.n_layers {
record(layer_file_name(layer));
}
let mut tensor_types: BTreeMap<String, String> = defs
.iter()
.map(|(name, def)| (name.clone(), def.dtype.to_owned()))
.collect();
for (layer, layout_layer) in layout.layers.iter().enumerate() {
for projection in &layout_layer.projections {
let stem = match projection.name {
ProjectionName::Gate => "ffn_gate_exps",
ProjectionName::Up => "ffn_up_exps",
ProjectionName::Down => "ffn_down_exps",
};
tensor_types.insert(
format!("blk.{layer}.{stem}.weight"),
projection.quant.clone(),
);
}
}
let manifest = Manifest {
rvmp_version: RVMP_VERSION,
model_id: g.model_id.to_owned(),
source: SourceInfo {
hf_repo: g.hf_repo.to_owned(),
revision: "deadbeef".to_owned(),
file: g.source_file.to_owned(),
sha256: "0".repeat(64),
},
arch: g.arch(),
quant: QuantInfo {
scheme: "gguf".to_owned(),
tensor_types,
},
common_tensors,
files,
};
write_manifest(&root, &manifest).expect("write manifest");
Fixture {
_tmp: tmp,
root,
manifest,
layout,
}
}
pub(crate) fn rewrite_layout(fixture: &mut Fixture, layout: &ExpertsLayout) {
write_layout(&fixture.root, layout).expect("rewrite layout");
let path = fixture.root.join(LAYOUT_FILE);
fixture.manifest.files.insert(
LAYOUT_FILE.to_owned(),
FileEntry {
size: fs::metadata(&path).unwrap().len(),
sha256: sha256_file(&path).unwrap(),
},
);
write_manifest(&fixture.root, &fixture.manifest).expect("rewrite manifest");
fixture.layout = layout.clone();
}