pub const QK_MXFP4: usize = 32;
pub const GGML_TYPE_MXFP4: u32 = 39;
pub const E2M1: [f32; 16] = [
0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0, -0.0, -0.5, -1.0, -1.5, -2.0, -3.0, -4.0, -6.0,
];
#[derive(Clone, Copy)]
#[repr(C)]
pub struct BlockMxfp4 {
pub e: u8,
pub qs: [u8; QK_MXFP4 / 2],
}
const _: () = assert!(std::mem::size_of::<BlockMxfp4>() == 17);
const _: () = assert!(std::mem::align_of::<BlockMxfp4>() == 1);
#[inline]
pub fn e8m0_to_f32(e: u8) -> Option<f32> {
match e {
0 => Some(f32::from_bits(0x0040_0000)),
0xFF => None,
_ => Some(f32::from_bits((e as u32) << 23)),
}
}
impl BlockMxfp4 {
#[inline]
pub fn dequantize(&self, out: &mut [f32; QK_MXFP4]) {
let Some(d) = e8m0_to_f32(self.e) else {
out.fill(0.0);
return;
};
for j in 0..QK_MXFP4 / 2 {
let byte = self.qs[j];
out[j] = d * E2M1[(byte & 0x0F) as usize];
out[j + QK_MXFP4 / 2] = d * E2M1[(byte >> 4) as usize];
}
}
}
pub fn dequantize(blocks: &[BlockMxfp4], out: &mut [f32]) -> crate::Result<()> {
if out.len() != blocks.len() * QK_MXFP4 {
return Err(crate::JoshuaError::Inference(format!(
"mxfp4: output holds {} values, expected {}",
out.len(),
blocks.len() * QK_MXFP4
)));
}
let mut scratch = [0f32; QK_MXFP4];
for (block, chunk) in blocks.iter().zip(out.chunks_exact_mut(QK_MXFP4)) {
block.dequantize(&mut scratch);
chunk.copy_from_slice(&scratch);
}
Ok(())
}
pub fn matmul_t(
(m, k, n): (usize, usize, usize),
lhs: &[f32],
rhs: &[BlockMxfp4],
dst: &mut [f32],
) -> crate::Result<()> {
let err = |msg: String| crate::JoshuaError::Inference(msg);
if !k.is_multiple_of(QK_MXFP4) {
return Err(err(format!(
"mxfp4 matmul: k={k} is not a multiple of the block size {QK_MXFP4}"
)));
}
let blocks_per_row = k / QK_MXFP4;
if rhs.len() != n * blocks_per_row {
return Err(err(format!(
"mxfp4 matmul: rhs holds {} blocks, expected {}",
rhs.len(),
n * blocks_per_row
)));
}
if lhs.len() != m * k || dst.len() != m * n {
return Err(err(format!(
"mxfp4 matmul: lhs/dst sized {}/{}, expected {}/{}",
lhs.len(),
dst.len(),
m * k,
m * n
)));
}
let mut decoded = [0f32; QK_MXFP4];
dst.fill(0.0);
for row in 0..n {
let row_blocks = &rhs[row * blocks_per_row..(row + 1) * blocks_per_row];
for (b, block) in row_blocks.iter().enumerate() {
block.dequantize(&mut decoded);
let base = b * QK_MXFP4;
for i in 0..m {
let a = &lhs[i * k + base..i * k + base + QK_MXFP4];
let mut acc = 0f32;
for (x, w) in a.iter().zip(decoded.iter()) {
acc += x * w;
}
dst[i * n + row] += acc;
}
}
}
Ok(())
}
pub fn blocks_from_bytes(bytes: &[u8]) -> crate::Result<&[BlockMxfp4]> {
let sz = std::mem::size_of::<BlockMxfp4>();
if !bytes.len().is_multiple_of(sz) {
return Err(crate::JoshuaError::Inference(format!(
"mxfp4: {} bytes is not a whole number of {sz}-byte blocks",
bytes.len()
)));
}
Ok(unsafe {
std::slice::from_raw_parts(bytes.as_ptr() as *const BlockMxfp4, bytes.len() / sz)
})
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn e8m0_decodes_to_powers_of_two() {
assert_eq!(e8m0_to_f32(127), Some(1.0));
assert_eq!(e8m0_to_f32(128), Some(2.0));
assert_eq!(e8m0_to_f32(126), Some(0.5));
assert_eq!(e8m0_to_f32(130), Some(8.0));
let tiny = e8m0_to_f32(0).unwrap();
assert!(tiny > 0.0, "e=0 must decode to 2^-127, not zero");
assert_eq!(tiny, f32::from_bits(0x0040_0000));
assert_eq!(e8m0_to_f32(0xFF), None);
}
#[test]
fn block_uses_split_half_nibble_order() {
let mut qs = [0u8; 16];
qs[0] = 0x7 | (0x1 << 4); let block = BlockMxfp4 { e: 127, qs }; let mut out = [0f32; QK_MXFP4];
block.dequantize(&mut out);
assert_eq!(out[0], 6.0, "low nibble of byte 0 is element 0");
assert_eq!(out[16], 0.5, "high nibble of byte 0 is element 16");
assert_eq!(out[1], 0.0, "element 1 comes from byte 1, not byte 0's high nibble");
}
#[test]
fn scale_multiplies_every_element() {
let mut qs = [0u8; 16];
qs[3] = 0xF; let block = BlockMxfp4 { e: 129, qs }; let mut out = [0f32; QK_MXFP4];
block.dequantize(&mut out);
assert_eq!(out[3], -24.0, "-6.0 * 4.0");
}
#[test]
fn nan_scale_decodes_to_zeros_not_infinities() {
let block = BlockMxfp4 {
e: 0xFF,
qs: [0xFF; 16],
};
let mut out = [0f32; QK_MXFP4];
block.dequantize(&mut out);
assert!(out.iter().all(|v| *v == 0.0), "corrupt scale must not poison the model");
}
#[test]
fn full_e2m1_table_round_trips_through_a_block() {
let mut qs = [0u8; 16];
for (j, q) in qs.iter_mut().enumerate() {
*q = j as u8;
}
let block = BlockMxfp4 { e: 127, qs };
let mut out = [0f32; QK_MXFP4];
block.dequantize(&mut out);
for j in 0..16 {
assert_eq!(out[j], E2M1[j], "code {j} must decode to its E2M1 value");
}
}
#[test]
fn matmul_matches_an_explicit_dequantized_reference() {
let (m, k, n) = (2usize, 64usize, 3usize);
let blocks_per_row = k / QK_MXFP4;
let mut rhs = Vec::new();
for r in 0..n {
for b in 0..blocks_per_row {
let mut qs = [0u8; 16];
for (j, q) in qs.iter_mut().enumerate() {
*q = (((j + r * 3 + b * 5) % 16) as u8) | ((((j + r) % 16) as u8) << 4);
}
rhs.push(BlockMxfp4 {
e: 127 + (r as u8 % 3),
qs,
});
}
}
let lhs: Vec<f32> = (0..m * k).map(|i| (i as f32 % 7.0) - 3.0).collect();
let mut dst = vec![0f32; m * n];
matmul_t((m, k, n), &lhs, &rhs, &mut dst).unwrap();
let mut weights = vec![0f32; n * k];
dequantize(&rhs, &mut weights).unwrap();
for i in 0..m {
for r in 0..n {
let expect: f32 = (0..k).map(|j| lhs[i * k + j] * weights[r * k + j]).sum();
let got = dst[i * n + r];
assert!(
(got - expect).abs() < 1e-3,
"matmul[{i},{r}] = {got}, reference {expect}"
);
}
}
}
#[test]
fn matmul_assigns_rather_than_accumulating_into_a_dirty_buffer() {
let (m, k, n) = (1usize, 32usize, 2usize);
let rhs = vec![
BlockMxfp4 {
e: 127,
qs: [0x21; 16],
},
BlockMxfp4 {
e: 128,
qs: [0x34; 16],
},
];
let lhs: Vec<f32> = (0..k).map(|i| (i % 5) as f32 - 2.0).collect();
let mut fresh = vec![0f32; m * n];
matmul_t((m, k, n), &lhs, &rhs, &mut fresh).unwrap();
let mut dirty = vec![123.75f32; m * n];
matmul_t((m, k, n), &lhs, &rhs, &mut dirty).unwrap();
assert_eq!(fresh, dirty, "a pre-filled destination must be overwritten");
}
#[test]
fn blocks_from_bytes_rejects_a_partial_block() {
assert!(blocks_from_bytes(&[0u8; 17]).is_ok());
assert!(blocks_from_bytes(&[0u8; 34]).is_ok());
assert!(blocks_from_bytes(&[0u8; 20]).is_err());
}
}