hf2q 0.1.23

Pure Rust CLI for converting HuggingFace models to hardware-optimized formats and serving them over an OpenAI-compatible API on Apple Silicon
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68

//! Enforce the ordinary Gemma matrix representation at the load boundary.

use anyhow::{ensure, Result};
use mlx_native::MlxBuffer;

use super::model::MlxModelWeights;
use crate::serve::forward_mlx_shared::MlxQWeight;

fn retain<'a>(weight: &'a MlxQWeight, buffers: &mut Vec<&'a MlxBuffer>) -> Result<()> {
    ensure!(
        weight.affine.is_none(),
        "ordinary GGUF matrix has an affine replacement"
    );
    ensure!(
        weight.f16_shadow.is_none(),
        "ordinary Gemma matrix has an F16 shadow"
    );
    buffers.push(&weight.buffer);
    Ok(())
}

impl MlxModelWeights {
    /// Matrix bytes are mapped unchanged. Elementwise norm/scaling state,
    /// activations, KV state and explicitly requested DWQ overlays have their
    /// own contracts and are excluded from this ordinary-load inventory.
    pub(super) fn verify_native_matrix_storage(&self) -> Result<()> {
        let mut buffers = Vec::new();
        retain(&self.embed_weight, &mut buffers)?;
        retain(self.resolved_lm_head(), &mut buffers)?;
        for layer in &self.layers {
            for projection in [
                &layer.attn.q_proj,
                &layer.attn.k_proj,
                &layer.attn.o_proj,
                &layer.mlp.gate_proj,
                &layer.mlp.up_proj,
                &layer.mlp.down_proj,
            ] {
                retain(projection, &mut buffers)?;
            }
            if let Some(v) = &layer.attn.v_proj {
                retain(v, &mut buffers)?;
            }
            match (&layer.moe.stacked_gate_up, &layer.moe.stacked_down) {
                (Some(gate_up), Some(down)) => {
                    retain(&layer.moe.router_proj, &mut buffers)?;
                    buffers.extend([gate_up, down]);
                }
                (None, None) => {}
                _ => anyhow::bail!("Gemma layer has an incomplete expert matrix pair"),
            }
        }
        let storage =
            crate::serve::native_matrix_storage::summarize_native_matrix_storage(buffers)?;
        ensure!(
            storage.anonymous_bytes == 0,
            "Gemma load created {} anonymous matrix bytes",
            storage.anonymous_bytes
        );
        tracing::info!(
            matrices = storage.unique_matrix_views,
            stored_matrix_bytes = storage.resident_bytes(),
            anonymous_matrix_bytes = storage.anonymous_bytes,
            "Gemma native matrix storage verified"
        );
        Ok(())
    }
}