Struct MXFP4Layer 

pub struct MXFP4Layer { /* private fields */ }

Implementations

impl MXFP4Layer

pub fn quantize( weight: &Tensor, bias: Option<Tensor>, device: &Device, ) -> Result<Arc<dyn QuantMethod>>

Quantize an unquantized weight tensor to MXFP4 format. weight shape: [N, K], bias shape: [N] (optional)

pub fn linear_b( in_dim: usize, out_dim: usize, config: &QuantizedConfig, bias: bool, vb: ShardedVarBuilder, ) -> Result<Arc<dyn QuantMethod>>

pub fn packed_linear_b( num_local_experts: usize, in_dim: usize, out_dim: usize, config: &QuantizedConfig, bias: bool, vb: ShardedVarBuilder, ) -> Result<Arc<dyn QuantMethod>>

pub fn packed_gptoss_linear( num_local_experts: usize, in_dim: usize, out_dim: usize, bias: bool, name: &str, vb: ShardedVarBuilder, ) -> Result<Arc<dyn QuantMethod>>

Load GPT-OSS style MXFP4 experts (combined gate_up_proj format).

GPT-OSS stores tensors as:

• {name}_blocks: [num_experts, out_dim, num_blocks, 16] where 16 bytes = 32 FP4 values
• {name}_scales: [num_experts, out_dim, num_blocks]
• {name}_bias: [num_experts, out_dim]

This function loads and reshapes the 4D blocks tensor to 3D [num_experts, out_dim, in_dim/2].

Trait Implementations

impl Debug for MXFP4Layer

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl QuantMethod for MXFP4Layer

fn new(method: QuantMethodConfig) -> Result<Self>

where Self: Sized,

fn dequantize_w(&self) -> Result<Tensor>

fn forward_raw(&self, x: &Tensor) -> Result<Tensor>

Raw matmul without dtype casting. Implementors override this. Callers should use forward instead.

fn gather_forward_raw(&self, x: &Tensor, indices: &Tensor) -> Result<Tensor>

Raw gather matmul without dtype casting. Implementors override this. Callers should use gather_forward instead.

fn quantized_act_type(&self) -> Option<DType>

If a quantized method, return the activation dtype.

fn add_delta_w(&self, _delta: &Tensor) -> Result<Arc<dyn QuantMethod>>

Add a delta weight from LoRA to the weights. This should be prescaled with alpha.

fn dtype_and_device(&self) -> (DType, Device)

Weight dtype and device

fn apply_isq( self: Arc<Self>, _dtype: Option<IsqType>, _device: Device, _n_quantized: &AtomicUsize, _imatrix_weight: Option<Vec<f32>>, _guard: QuantizeOntoGuard, ) -> Result<Arc<dyn QuantMethod>>

If the quant is backed by a qmatmul.

fn forward(&self, a: &Tensor) -> Result<Tensor>

Compute matmul of self and a. self should contain the weights. Automatically casts to the required quantization activation type and back.

fn gather_forward(&self, a: &Tensor, indices: &Tensor) -> Result<Tensor>

Compute gather matmul of self and a. self should contain the weights. Automatically casts to the required quantization activation type and back.

fn afq_inner(&self) -> Option<AfqInner<'_>>

If this is an AFQ layer, return its (w_q, scales, biases, bits, group_size). Used by Metal fused QKV / gate-up paths.

fn unquant_weight_bias(&self) -> Option<(Tensor, Option<Tensor>)>

fn has_bias(&self) -> bool

fn begin_track_stats(&mut self) -> Result<()>

Begin tracking stats into an ImatrixLayerStats

fn end_track_stats(&self) -> Result<Tensor>

End tracking stats into an ImatrixLayerStats. Returns the computed imatrix.

fn is_distributed(&self) -> Option<DistributedKind>

fn dummy_info(&self) -> Option<&DummyLayerInfo>

impl QuantizedSerde for MXFP4Layer

fn name(&self) -> &'static str

fn isq_serde_supported(&self) -> bool

fn serialize(&self) -> Result<Cow<'_, [u8]>>

fn serialize_with_bias(&self, bias: Option<Tensor>) -> Result<Cow<'_, [u8]>>

NOT meant for external calling

fn deserialize( data: Cow<'_, [u8]>, device: &Device, _comm: &Arc<Comm>, guard: QuantizeOntoGuard, ) -> Result<Arc<dyn QuantMethod>>

where Self: Sized,

fn deserialize_ext_bias( data: Cow<'_, [u8]>, device: &Device, guard: QuantizeOntoGuard, ) -> Result<(Arc<dyn QuantMethod>, Option<Tensor>)>

where Self: Sized,