Skip to main content

trueno/inference/
model.rs

1//! Llama-family transformer model for inference.
2//!
3//! Composes trueno primitives (rms_norm, Q4K matmul, fused attention)
4//! into a complete transformer that loads GGUF weights and generates text.
5
6use crate::backends::q4k::matmul_q4k_f32_dispatch;
7use crate::blis::attention::fused_attention_decode;
8use crate::blis::norms::rms_norm;
9use crate::error::TruenoError;
10use crate::inference::gguf::{GgmlType, GgufFile};
11
12/// Model hyperparameters extracted from GGUF metadata.
13#[derive(Debug, Clone)]
14pub struct ModelConfig {
15    pub hidden_size: usize,
16    pub intermediate_size: usize,
17    pub num_layers: usize,
18    pub num_heads: usize,
19    pub num_kv_heads: usize,
20    pub head_dim: usize,
21    pub vocab_size: usize,
22    pub rms_norm_eps: f32,
23    pub rope_theta: f32,
24    pub max_seq_len: usize,
25    pub arch: String,
26}
27
28impl ModelConfig {
29    /// Extract config from GGUF metadata.
30    pub fn from_gguf(gguf: &GgufFile) -> Result<Self, TruenoError> {
31        let arch = gguf.meta_str("general.architecture").unwrap_or("llama").to_string();
32        let prefix = &arch; // e.g., "llama" or "qwen2"
33
34        let hidden_size = gguf
35            .meta_u32(&format!("{prefix}.embedding_length"))
36            .ok_or_else(|| TruenoError::InvalidInput("Missing embedding_length in GGUF".into()))?
37            as usize;
38
39        let num_heads = gguf
40            .meta_u32(&format!("{prefix}.attention.head_count"))
41            .ok_or_else(|| TruenoError::InvalidInput("Missing head_count in GGUF".into()))?
42            as usize;
43
44        let num_kv_heads = gguf
45            .meta_u32(&format!("{prefix}.attention.head_count_kv"))
46            .unwrap_or(num_heads as u32) as usize;
47
48        let num_layers = gguf
49            .meta_u32(&format!("{prefix}.block_count"))
50            .ok_or_else(|| TruenoError::InvalidInput("Missing block_count in GGUF".into()))?
51            as usize;
52
53        let intermediate_size =
54            gguf.meta_u32(&format!("{prefix}.feed_forward_length")).ok_or_else(|| {
55                TruenoError::InvalidInput("Missing feed_forward_length in GGUF".into())
56            })? as usize;
57
58        let head_dim = hidden_size / num_heads;
59
60        let vocab_size = gguf
61            .meta_u32("tokenizer.ggml.vocab_size")
62            .or_else(|| {
63                // Fallback: count tokens array
64                gguf.metadata.get("tokenizer.ggml.tokens").and_then(|v| {
65                    if let crate::inference::gguf::MetadataValue::Array(arr) = v {
66                        Some(arr.len() as u32)
67                    } else {
68                        None
69                    }
70                })
71            })
72            .unwrap_or(32000) as usize;
73
74        let rms_norm_eps =
75            gguf.meta_f32(&format!("{prefix}.attention.layer_norm_rms_epsilon")).unwrap_or(1e-5);
76
77        let rope_theta = gguf.meta_f32(&format!("{prefix}.rope.freq_base")).unwrap_or(10000.0);
78
79        let max_seq_len =
80            gguf.meta_u32(&format!("{prefix}.context_length")).unwrap_or(2048) as usize;
81
82        Ok(Self {
83            hidden_size,
84            intermediate_size,
85            num_layers,
86            num_heads,
87            num_kv_heads,
88            head_dim,
89            vocab_size,
90            rms_norm_eps,
91            rope_theta,
92            max_seq_len,
93            arch,
94        })
95    }
96}
97
98/// A weight matrix that may be Q4K (bytes) or any-other-quant dequantized to F32.
99pub enum WeightMatrix {
100    /// Raw Q4K bytes — use matmul_q4k_f32_dispatch
101    Q4K { data: Vec<u8>, rows: usize },
102    /// Dequantized F32 — use scalar dot-product
103    F32 { data: Vec<f32>, rows: usize },
104}
105
106impl WeightMatrix {
107    pub fn rows(&self) -> usize {
108        match self {
109            WeightMatrix::Q4K { rows, .. } => *rows,
110            WeightMatrix::F32 { rows, .. } => *rows,
111        }
112    }
113}
114
115/// Weights for a single transformer layer.
116pub struct LayerWeights {
117    // Attention
118    pub attn_norm: Vec<f32>,
119    pub q_weight: WeightMatrix,
120    pub k_weight: WeightMatrix,
121    pub v_weight: WeightMatrix,
122    pub o_weight: WeightMatrix,
123    // Qwen2/Qwen3 biases (None for LLaMA)
124    pub q_bias: Option<Vec<f32>>,
125    pub k_bias: Option<Vec<f32>>,
126    pub v_bias: Option<Vec<f32>>,
127
128    // FFN
129    pub ffn_norm: Vec<f32>,
130    pub gate_weight: WeightMatrix,
131    pub up_weight: WeightMatrix,
132    pub down_weight: WeightMatrix,
133}
134
135/// Full model weights.
136pub struct ModelWeights {
137    pub token_embd: Vec<f32>,  // [vocab_size, hidden_size]
138    pub output_norm: Vec<f32>, // [hidden_size]
139    pub output_weight: WeightMatrix,
140    pub layers: Vec<LayerWeights>,
141}
142
143/// Pre-allocated scratch buffers for the forward pass.
144/// Eliminates all per-token heap allocations (FALSIFY-ARENA-001).
145/// Contract: contracts/cgp/cgp-inference-arena-v1.yaml
146pub struct ForwardArena {
147    pub attn_input: Vec<f32>,
148    pub q: Vec<f32>,
149    pub k: Vec<f32>,
150    pub v: Vec<f32>,
151    pub attn_out: Vec<f32>,
152    pub attn_proj: Vec<f32>,
153    pub ffn_input: Vec<f32>,
154    pub gate: Vec<f32>,
155    pub up: Vec<f32>,
156    pub swiglu: Vec<f32>,
157    pub ffn_out: Vec<f32>,
158    pub hidden: Vec<f32>,
159    pub residual: Vec<f32>,
160    pub normed: Vec<f32>,
161    pub k_cache_head: Vec<f32>,
162    pub v_cache_head: Vec<f32>,
163}
164
165impl ForwardArena {
166    pub fn new(config: &ModelConfig) -> Self {
167        let kv_dim = config.num_kv_heads * config.head_dim;
168        let head_cache_size = config.max_seq_len * config.head_dim;
169        Self {
170            attn_input: vec![0.0f32; config.hidden_size],
171            q: vec![0.0f32; config.hidden_size],
172            k: vec![0.0f32; kv_dim],
173            v: vec![0.0f32; kv_dim],
174            attn_out: vec![0.0f32; config.hidden_size],
175            attn_proj: vec![0.0f32; config.hidden_size],
176            ffn_input: vec![0.0f32; config.hidden_size],
177            gate: vec![0.0f32; config.intermediate_size],
178            up: vec![0.0f32; config.intermediate_size],
179            swiglu: vec![0.0f32; config.intermediate_size],
180            ffn_out: vec![0.0f32; config.hidden_size],
181            hidden: vec![0.0f32; config.hidden_size],
182            residual: vec![0.0f32; config.hidden_size],
183            normed: vec![0.0f32; config.hidden_size],
184            k_cache_head: vec![0.0f32; head_cache_size],
185            v_cache_head: vec![0.0f32; head_cache_size],
186        }
187    }
188}
189
190/// KV cache for incremental decoding.
191pub struct KvCache {
192    /// k_cache[layer][pos * head_dim * num_kv_heads .. ] — flat per-layer
193    pub k: Vec<Vec<f32>>,
194    /// v_cache[layer][pos * head_dim * num_kv_heads .. ]
195    pub v: Vec<Vec<f32>>,
196    pub seq_len: usize,
197}
198
199impl KvCache {
200    pub fn new(config: &ModelConfig) -> Self {
201        let kv_dim = config.num_kv_heads * config.head_dim;
202        let layer_size = config.max_seq_len * kv_dim;
203        Self {
204            k: (0..config.num_layers).map(|_| vec![0.0f32; layer_size]).collect(),
205            v: (0..config.num_layers).map(|_| vec![0.0f32; layer_size]).collect(),
206            seq_len: 0,
207        }
208    }
209}
210
211/// Complete transformer model ready for inference.
212pub struct LlamaModel {
213    pub config: ModelConfig,
214    pub weights: ModelWeights,
215}
216
217impl LlamaModel {
218    /// Load model from a GGUF file.
219    pub fn from_gguf(gguf: &GgufFile) -> Result<Self, TruenoError> {
220        let config = ModelConfig::from_gguf(gguf)?;
221
222        eprintln!(
223            "Loading {} model: {}L × {}H ({}h {}kv) × {}I, vocab={}",
224            config.arch,
225            config.num_layers,
226            config.hidden_size,
227            config.num_heads,
228            config.num_kv_heads,
229            config.intermediate_size,
230            config.vocab_size,
231        );
232
233        let weights = load_weights(gguf, &config)?;
234
235        Ok(Self { config, weights })
236    }
237
238    /// Run one forward pass for a single token at the given position.
239    /// Returns logits [vocab_size].
240    /// Uses ForwardArena for zero per-token allocations (FALSIFY-ARENA-001).
241    pub fn forward(
242        &self,
243        token_id: u32,
244        pos: usize,
245        kv_cache: &mut KvCache,
246        arena: &mut ForwardArena,
247    ) -> Result<Vec<f32>, TruenoError> {
248        let cfg = &self.config;
249        let w = &self.weights;
250
251        // Token embedding lookup (copy into arena.hidden)
252        let embd_start = token_id as usize * cfg.hidden_size;
253        let embd_end = embd_start + cfg.hidden_size;
254        if embd_end > w.token_embd.len() {
255            return Err(TruenoError::InvalidInput(format!(
256                "Token ID {token_id} out of range (vocab={})",
257                cfg.vocab_size
258            )));
259        }
260        arena.hidden[..cfg.hidden_size].copy_from_slice(&w.token_embd[embd_start..embd_end]);
261
262        // Transformer layers
263        for (layer_idx, lw) in w.layers.iter().enumerate() {
264            self.forward_layer(layer_idx, lw, pos, kv_cache, arena)?;
265        }
266
267        // Final RMS norm
268        rms_norm(
269            &arena.hidden[..cfg.hidden_size],
270            &w.output_norm,
271            cfg.rms_norm_eps,
272            &mut arena.normed[..cfg.hidden_size],
273        )?;
274
275        // Output projection → logits (this one allocation is unavoidable — vocab_size is large)
276        let logits =
277            matmul_weight(&w.output_weight, &arena.normed[..cfg.hidden_size], cfg.hidden_size);
278
279        Ok(logits)
280    }
281
282    /// Forward one layer, reading/writing arena.hidden in-place.
283    fn forward_layer(
284        &self,
285        layer_idx: usize,
286        lw: &LayerWeights,
287        pos: usize,
288        kv_cache: &mut KvCache,
289        arena: &mut ForwardArena,
290    ) -> Result<(), TruenoError> {
291        let cfg = &self.config;
292        let kv_dim = cfg.num_kv_heads * cfg.head_dim;
293        let h_sz = cfg.hidden_size;
294
295        // === Attention block ===
296        rms_norm(
297            &arena.hidden[..h_sz],
298            &lw.attn_norm,
299            cfg.rms_norm_eps,
300            &mut arena.attn_input[..h_sz],
301        )?;
302
303        // QKV projections into arena buffers
304        matmul_weight_into(&lw.q_weight, &arena.attn_input[..h_sz], h_sz, &mut arena.q[..h_sz]);
305        matmul_weight_into(&lw.k_weight, &arena.attn_input[..h_sz], h_sz, &mut arena.k[..kv_dim]);
306        matmul_weight_into(&lw.v_weight, &arena.attn_input[..h_sz], h_sz, &mut arena.v[..kv_dim]);
307
308        // Optional bias (Qwen2/Qwen3)
309        if let Some(bias) = &lw.q_bias {
310            for (v, b) in arena.q[..h_sz].iter_mut().zip(bias.iter()) {
311                *v += b;
312            }
313        }
314        if let Some(bias) = &lw.k_bias {
315            for (v, b) in arena.k[..kv_dim].iter_mut().zip(bias.iter()) {
316                *v += b;
317            }
318        }
319        if let Some(bias) = &lw.v_bias {
320            for (v, b) in arena.v[..kv_dim].iter_mut().zip(bias.iter()) {
321                *v += b;
322            }
323        }
324
325        // RoPE in-place
326        apply_rope(&mut arena.q[..h_sz], cfg.num_heads, cfg.head_dim, pos, cfg.rope_theta);
327        apply_rope(&mut arena.k[..kv_dim], cfg.num_kv_heads, cfg.head_dim, pos, cfg.rope_theta);
328
329        // Store K,V in cache
330        let kv_off = pos * kv_dim;
331        kv_cache.k[layer_idx][kv_off..kv_off + kv_dim].copy_from_slice(&arena.k[..kv_dim]);
332        kv_cache.v[layer_idx][kv_off..kv_off + kv_dim].copy_from_slice(&arena.v[..kv_dim]);
333
334        let seq_len = pos + 1;
335
336        // Multi-head attention
337        arena.attn_out[..h_sz].fill(0.0);
338        let heads_per_kv = cfg.num_heads / cfg.num_kv_heads;
339
340        for h in 0..cfg.num_heads {
341            let kv_h = h / heads_per_kv;
342            let q_head = &arena.q[h * cfg.head_dim..(h + 1) * cfg.head_dim];
343
344            // Build contiguous K/V view for this head (reuse arena buffers)
345            let view_len = seq_len * cfg.head_dim;
346            for s in 0..seq_len {
347                let src_off = s * kv_dim + kv_h * cfg.head_dim;
348                let dst_off = s * cfg.head_dim;
349                arena.k_cache_head[dst_off..dst_off + cfg.head_dim]
350                    .copy_from_slice(&kv_cache.k[layer_idx][src_off..src_off + cfg.head_dim]);
351                arena.v_cache_head[dst_off..dst_off + cfg.head_dim]
352                    .copy_from_slice(&kv_cache.v[layer_idx][src_off..src_off + cfg.head_dim]);
353            }
354
355            let out_head = &mut arena.attn_out[h * cfg.head_dim..(h + 1) * cfg.head_dim];
356            fused_attention_decode(
357                q_head,
358                &arena.k_cache_head[..view_len],
359                &arena.v_cache_head[..view_len],
360                cfg.head_dim,
361                seq_len,
362                out_head,
363            );
364        }
365
366        // Output projection
367        matmul_weight_into(
368            &lw.o_weight,
369            &arena.attn_out[..h_sz],
370            h_sz,
371            &mut arena.attn_proj[..h_sz],
372        );
373
374        // Residual: residual = hidden + attn_proj
375        for i in 0..h_sz {
376            arena.residual[i] = arena.hidden[i] + arena.attn_proj[i];
377        }
378
379        // === FFN block ===
380        rms_norm(
381            &arena.residual[..h_sz],
382            &lw.ffn_norm,
383            cfg.rms_norm_eps,
384            &mut arena.ffn_input[..h_sz],
385        )?;
386
387        let i_sz = cfg.intermediate_size;
388        matmul_weight_into(
389            &lw.gate_weight,
390            &arena.ffn_input[..h_sz],
391            h_sz,
392            &mut arena.gate[..i_sz],
393        );
394        matmul_weight_into(&lw.up_weight, &arena.ffn_input[..h_sz], h_sz, &mut arena.up[..i_sz]);
395
396        // SiLU(gate) * up → swiglu
397        for i in 0..i_sz {
398            let g = arena.gate[i];
399            let silu_g = g / (1.0 + (-g).exp());
400            arena.swiglu[i] = silu_g * arena.up[i];
401        }
402
403        // Down projection
404        matmul_weight_into(
405            &lw.down_weight,
406            &arena.swiglu[..i_sz],
407            i_sz,
408            &mut arena.ffn_out[..h_sz],
409        );
410
411        // Residual → hidden (for next layer)
412        for i in 0..h_sz {
413            arena.hidden[i] = arena.residual[i] + arena.ffn_out[i];
414        }
415
416        Ok(())
417    }
418}
419
420/// Apply Rotary Position Embedding (RoPE) in-place.
421fn apply_rope(x: &mut [f32], num_heads: usize, head_dim: usize, pos: usize, theta: f32) {
422    for h in 0..num_heads {
423        let head = &mut x[h * head_dim..(h + 1) * head_dim];
424        for i in (0..head_dim).step_by(2) {
425            let freq = 1.0 / theta.powf(i as f32 / head_dim as f32);
426            let angle = pos as f32 * freq;
427            let (sin_a, cos_a) = angle.sin_cos();
428            let x0 = head[i];
429            let x1 = head[i + 1];
430            head[i] = x0 * cos_a - x1 * sin_a;
431            head[i + 1] = x0 * sin_a + x1 * cos_a;
432        }
433    }
434}
435
436/// Load all weights from GGUF into model weight structs.
437fn load_weights(gguf: &GgufFile, config: &ModelConfig) -> Result<ModelWeights, TruenoError> {
438    // Token embeddings — may be F32, F16, or quantized (Q4K/Q6K).
439    // For quantized embeddings, dequantize the full table at load time
440    // since we need random-access per-token lookup.
441    let token_embd = load_f32_or_dequant_tensor(
442        gguf,
443        "token_embd.weight",
444        config.vocab_size * config.hidden_size,
445    )?;
446
447    // Output norm
448    let output_norm = load_f32_tensor(gguf, "output_norm.weight", config.hidden_size)?;
449
450    // Output projection — Q4K kept as bytes; everything else dequantized to F32.
451    // Falls back to tied embeddings if output.weight not present.
452    let output_weight = if gguf.tensor_info("output.weight").is_some() {
453        load_weight_matrix(gguf, "output.weight", config.hidden_size)?
454    } else {
455        // Tied embeddings
456        WeightMatrix::F32 { data: token_embd.clone(), rows: config.vocab_size }
457    };
458
459    // Layers
460    let mut layers = Vec::with_capacity(config.num_layers);
461    for i in 0..config.num_layers {
462        let prefix = format!("blk.{i}");
463
464        let attn_norm =
465            load_f32_tensor(gguf, &format!("{prefix}.attn_norm.weight"), config.hidden_size)?;
466        let ffn_norm =
467            load_f32_tensor(gguf, &format!("{prefix}.ffn_norm.weight"), config.hidden_size)?;
468
469        let q_weight =
470            load_weight_matrix(gguf, &format!("{prefix}.attn_q.weight"), config.hidden_size)?;
471        let k_weight =
472            load_weight_matrix(gguf, &format!("{prefix}.attn_k.weight"), config.hidden_size)?;
473        let v_weight =
474            load_weight_matrix(gguf, &format!("{prefix}.attn_v.weight"), config.hidden_size)?;
475        let o_weight =
476            load_weight_matrix(gguf, &format!("{prefix}.attn_output.weight"), config.hidden_size)?;
477
478        // Qwen2/Qwen3 attention biases (optional — LLaMA has none)
479        let kv_dim = config.num_kv_heads * config.head_dim;
480        let q_bias = load_optional_f32(gguf, &format!("{prefix}.attn_q.bias"), config.hidden_size);
481        let k_bias = load_optional_f32(gguf, &format!("{prefix}.attn_k.bias"), kv_dim);
482        let v_bias = load_optional_f32(gguf, &format!("{prefix}.attn_v.bias"), kv_dim);
483
484        let gate_weight =
485            load_weight_matrix(gguf, &format!("{prefix}.ffn_gate.weight"), config.hidden_size)?;
486        let up_weight =
487            load_weight_matrix(gguf, &format!("{prefix}.ffn_up.weight"), config.hidden_size)?;
488        let down_weight = load_weight_matrix(
489            gguf,
490            &format!("{prefix}.ffn_down.weight"),
491            config.intermediate_size,
492        )?;
493
494        if i == 0 {
495            eprintln!(
496                "  Layer 0: Q[{}×{}] K[{}×{}] V[{}×{}] Gate[{}×{}]",
497                q_weight.rows(),
498                config.hidden_size,
499                k_weight.rows(),
500                config.hidden_size,
501                v_weight.rows(),
502                config.hidden_size,
503                gate_weight.rows(),
504                config.hidden_size,
505            );
506        }
507
508        layers.push(LayerWeights {
509            attn_norm,
510            q_weight,
511            k_weight,
512            v_weight,
513            o_weight,
514            q_bias,
515            k_bias,
516            v_bias,
517            ffn_norm,
518            gate_weight,
519            up_weight,
520            down_weight,
521        });
522    }
523
524    eprintln!("  Loaded {} layers", layers.len());
525
526    Ok(ModelWeights { token_embd, output_norm, output_weight, layers })
527}
528
529/// Load a tensor as F32, dequantizing if quantized.
530/// For Q4K weights, uses trueno's dequantize_q4k_to_f32.
531#[deny(clippy::wildcard_enum_match_arm)]
532fn load_f32_or_dequant_tensor(
533    gguf: &GgufFile,
534    name: &str,
535    expected_elements: usize,
536) -> Result<Vec<f32>, TruenoError> {
537    let info = gguf
538        .tensor_info(name)
539        .ok_or_else(|| TruenoError::InvalidInput(format!("Missing tensor: {name}")))?;
540    let data = gguf
541        .tensor_data(name)
542        .ok_or_else(|| TruenoError::InvalidInput(format!("Missing tensor data: {name}")))?;
543
544    match info.dtype {
545        GgmlType::F32 | GgmlType::F16 | GgmlType::BF16 => {
546            Ok(to_f32_from_any(data, info.dtype, expected_elements))
547        }
548        GgmlType::Q4K => {
549            let n_elements = info.n_elements() as usize;
550            Ok(crate::backends::q4k::dequantize_q4k_to_f32(data, n_elements))
551        }
552        GgmlType::Q6K => Ok(dequantize_q6k_to_f32(data, info.n_elements() as usize)),
553        GgmlType::Q5K => Ok(dequantize_q5k_to_f32(data, info.n_elements() as usize)),
554        GgmlType::Q8_0 => Ok(dequantize_q8_0_to_f32(data, info.n_elements() as usize)),
555        GgmlType::Q4_0 => Ok(dequantize_q4_0_to_f32(data, info.n_elements() as usize)),
556        GgmlType::Q4_1 => Ok(dequantize_q4_1_to_f32(data, info.n_elements() as usize)),
557        // Named, not `_` (#3431): a new upstream ggml id must fail to compile here.
558        GgmlType::Q5_0
559        | GgmlType::Q5_1
560        | GgmlType::Q8_1
561        | GgmlType::Q2K
562        | GgmlType::Q3K
563        | GgmlType::Q8K
564        | GgmlType::IQ2XXS
565        | GgmlType::IQ2XS
566        | GgmlType::IQ3XXS
567        | GgmlType::IQ1S
568        | GgmlType::IQ4NL
569        | GgmlType::IQ3S
570        | GgmlType::IQ2S
571        | GgmlType::IQ4XS
572        | GgmlType::I8
573        | GgmlType::I16
574        | GgmlType::I32
575        | GgmlType::I64
576        | GgmlType::F64
577        | GgmlType::IQ1M
578        | GgmlType::TQ1_0
579        | GgmlType::TQ2_0
580        | GgmlType::MXFP4
581        | GgmlType::NVFP4
582        | GgmlType::Q1_0
583        | GgmlType::Q2_0 => {
584            eprintln!(
585                "  WARNING: tensor '{name}' has unsupported dtype {:?}, using zeros",
586                info.dtype
587            );
588            Ok(vec![0.0f32; expected_elements])
589        }
590    }
591}
592
593/// Load an optional F32 tensor (returns None if tensor doesn't exist in GGUF).
594fn load_optional_f32(gguf: &GgufFile, name: &str, expected_elements: usize) -> Option<Vec<f32>> {
595    let info = gguf.tensor_info(name)?;
596    let data = gguf.tensor_data(name)?;
597    Some(to_f32_from_any(data, info.dtype, expected_elements))
598}
599
600/// Load a tensor as F32 (dequantizing F16 if needed).
601fn load_f32_tensor(
602    gguf: &GgufFile,
603    name: &str,
604    expected_elements: usize,
605) -> Result<Vec<f32>, TruenoError> {
606    let info = gguf
607        .tensor_info(name)
608        .ok_or_else(|| TruenoError::InvalidInput(format!("Missing tensor: {name}")))?;
609    let data = gguf
610        .tensor_data(name)
611        .ok_or_else(|| TruenoError::InvalidInput(format!("Missing tensor data: {name}")))?;
612
613    Ok(to_f32_from_any(data, info.dtype, expected_elements))
614}
615
616/// Load a weight tensor as a `WeightMatrix`.
617/// Q4K weights are kept as raw bytes for the fused matmul kernel.
618/// All other quantization types are dequantized to F32 at load time.
619#[deny(clippy::wildcard_enum_match_arm)]
620fn load_weight_matrix(
621    gguf: &GgufFile,
622    name: &str,
623    in_dim: usize,
624) -> Result<WeightMatrix, TruenoError> {
625    let info = gguf
626        .tensor_info(name)
627        .ok_or_else(|| TruenoError::InvalidInput(format!("Missing tensor: {name}")))?;
628    let data = gguf
629        .tensor_data(name)
630        .ok_or_else(|| TruenoError::InvalidInput(format!("Missing tensor data: {name}")))?;
631
632    let n_elements = info.n_elements() as usize;
633    let out_dim = n_elements / in_dim;
634
635    match info.dtype {
636        GgmlType::Q4K => Ok(WeightMatrix::Q4K { data: data.to_vec(), rows: out_dim }),
637        GgmlType::F32 | GgmlType::F16 | GgmlType::BF16 => {
638            let f32_data = to_f32_from_any(data, info.dtype, n_elements);
639            Ok(WeightMatrix::F32 { data: f32_data, rows: out_dim })
640        }
641        GgmlType::Q6K => {
642            let f32_data = dequantize_q6k_to_f32(data, n_elements);
643            Ok(WeightMatrix::F32 { data: f32_data, rows: out_dim })
644        }
645        GgmlType::Q5K => {
646            let f32_data = dequantize_q5k_to_f32(data, n_elements);
647            Ok(WeightMatrix::F32 { data: f32_data, rows: out_dim })
648        }
649        GgmlType::Q8_0 => {
650            let f32_data = dequantize_q8_0_to_f32(data, n_elements);
651            Ok(WeightMatrix::F32 { data: f32_data, rows: out_dim })
652        }
653        GgmlType::Q4_0 => {
654            let f32_data = dequantize_q4_0_to_f32(data, n_elements);
655            Ok(WeightMatrix::F32 { data: f32_data, rows: out_dim })
656        }
657        GgmlType::Q4_1 => {
658            let f32_data = dequantize_q4_1_to_f32(data, n_elements);
659            Ok(WeightMatrix::F32 { data: f32_data, rows: out_dim })
660        }
661        // Named, not `_` (#3431): a new upstream ggml id must fail to compile here.
662        GgmlType::Q5_0
663        | GgmlType::Q5_1
664        | GgmlType::Q8_1
665        | GgmlType::Q2K
666        | GgmlType::Q3K
667        | GgmlType::Q8K
668        | GgmlType::IQ2XXS
669        | GgmlType::IQ2XS
670        | GgmlType::IQ3XXS
671        | GgmlType::IQ1S
672        | GgmlType::IQ4NL
673        | GgmlType::IQ3S
674        | GgmlType::IQ2S
675        | GgmlType::IQ4XS
676        | GgmlType::I8
677        | GgmlType::I16
678        | GgmlType::I32
679        | GgmlType::I64
680        | GgmlType::F64
681        | GgmlType::IQ1M
682        | GgmlType::TQ1_0
683        | GgmlType::TQ2_0
684        | GgmlType::MXFP4
685        | GgmlType::NVFP4
686        | GgmlType::Q1_0
687        | GgmlType::Q2_0 => {
688            eprintln!("  WARNING: tensor '{name}' dtype {:?} unsupported, using zeros", info.dtype);
689            Ok(WeightMatrix::F32 { data: vec![0.0f32; n_elements], rows: out_dim })
690        }
691    }
692}
693
694/// Dispatch matrix-vector multiply based on weight type (allocating).
695/// Used only for output projection (vocab_size too large for arena).
696fn matmul_weight(weight: &WeightMatrix, input: &[f32], in_dim: usize) -> Vec<f32> {
697    match weight {
698        WeightMatrix::Q4K { data, rows } => matmul_q4k_f32_dispatch(data, input, *rows, in_dim),
699        WeightMatrix::F32 { data, rows } => {
700            let mut out = vec![0.0f32; *rows];
701            for i in 0..*rows {
702                let row = &data[i * in_dim..(i + 1) * in_dim];
703                out[i] = row.iter().zip(input.iter()).map(|(a, b)| a * b).sum();
704            }
705            out
706        }
707    }
708}
709
710/// Dispatch matrix-vector multiply into a pre-allocated output buffer (zero-alloc).
711fn matmul_weight_into(weight: &WeightMatrix, input: &[f32], in_dim: usize, out: &mut [f32]) {
712    match weight {
713        WeightMatrix::Q4K { data, rows } => {
714            let result = matmul_q4k_f32_dispatch(data, input, *rows, in_dim);
715            out[..*rows].copy_from_slice(&result);
716        }
717        WeightMatrix::F32 { data, rows } => {
718            for i in 0..*rows {
719                let row = &data[i * in_dim..(i + 1) * in_dim];
720                out[i] = row.iter().zip(input.iter()).map(|(a, b)| a * b).sum();
721            }
722        }
723    }
724}
725
726/// Convert IEEE 754 half-precision (FP16) bits to f32.
727fn f16_to_f32(bits: u16) -> f32 {
728    let sign = ((bits >> 15) as u32) << 31;
729    let exp = ((bits >> 10) & 0x1F) as u32;
730    let mant = (bits & 0x3FF) as u32;
731
732    if exp == 0 {
733        if mant == 0 {
734            return f32::from_bits(sign); // ±0
735        }
736        // Denormalized: convert to normalized f32
737        let mut m = mant;
738        let mut e: i32 = -14;
739        while m & 0x400 == 0 {
740            m <<= 1;
741            e -= 1;
742        }
743        m &= 0x3FF;
744        let f32_exp = ((e + 127) as u32) << 23;
745        return f32::from_bits(sign | f32_exp | (m << 13));
746    }
747    if exp == 31 {
748        // Inf/NaN
749        return f32::from_bits(sign | 0x7F80_0000 | (mant << 13));
750    }
751    let f32_exp = (exp + 112) << 23; // rebias: -15 + 127 = 112
752    f32::from_bits(sign | f32_exp | (mant << 13))
753}
754
755/// Convert tensor bytes to f32, handling F32, F16, BF16.
756#[deny(clippy::wildcard_enum_match_arm)]
757fn to_f32_from_any(data: &[u8], dtype: GgmlType, n_elements: usize) -> Vec<f32> {
758    match dtype {
759        GgmlType::F32 => {
760            // Safe: read f32 values from aligned-or-unaligned bytes
761            let count = n_elements.min(data.len() / 4);
762            (0..count)
763                .map(|i| {
764                    let off = i * 4;
765                    f32::from_le_bytes([data[off], data[off + 1], data[off + 2], data[off + 3]])
766                })
767                .collect()
768        }
769        GgmlType::F16 => {
770            let count = n_elements.min(data.len() / 2);
771            (0..count)
772                .map(|i| {
773                    let off = i * 2;
774                    let bits = u16::from_le_bytes([data[off], data[off + 1]]);
775                    f16_to_f32(bits)
776                })
777                .collect()
778        }
779        GgmlType::BF16 => {
780            let count = n_elements.min(data.len() / 2);
781            (0..count)
782                .map(|i| {
783                    let off = i * 2;
784                    let bits = u16::from_le_bytes([data[off], data[off + 1]]);
785                    f32::from_bits((bits as u32) << 16)
786                })
787                .collect()
788        }
789        // Named, not `_` (#3431): a new upstream ggml id must fail to compile here.
790        GgmlType::Q4_0
791        | GgmlType::Q4_1
792        | GgmlType::Q5_0
793        | GgmlType::Q5_1
794        | GgmlType::Q8_0
795        | GgmlType::Q8_1
796        | GgmlType::Q2K
797        | GgmlType::Q3K
798        | GgmlType::Q4K
799        | GgmlType::Q5K
800        | GgmlType::Q6K
801        | GgmlType::Q8K
802        | GgmlType::IQ2XXS
803        | GgmlType::IQ2XS
804        | GgmlType::IQ3XXS
805        | GgmlType::IQ1S
806        | GgmlType::IQ4NL
807        | GgmlType::IQ3S
808        | GgmlType::IQ2S
809        | GgmlType::IQ4XS
810        | GgmlType::I8
811        | GgmlType::I16
812        | GgmlType::I32
813        | GgmlType::I64
814        | GgmlType::F64
815        | GgmlType::IQ1M
816        | GgmlType::TQ1_0
817        | GgmlType::TQ2_0
818        | GgmlType::MXFP4
819        | GgmlType::NVFP4
820        | GgmlType::Q1_0
821        | GgmlType::Q2_0 => {
822            // For quantized norms (shouldn't happen), return zeros
823            vec![0.0f32; n_elements]
824        }
825    }
826}
827
828/// Dequantize Q6_K to F32.
829///
830/// Q6_K layout per 256-element super-block (210 bytes):
831/// - ql[128]: lower 4 bits of each 6-bit value (2 values per byte)
832/// - qh[64]:  upper 2 bits (4 values per byte, 2 bits each)
833/// - scales[16]: signed 8-bit scales for 16 groups of 16
834/// - d[2]: f16 global scale
835///
836/// Value = d * scale[group] * q6  where q6 = (low4 | high2<<4) as i8 - 32
837fn dequantize_q6k_to_f32(data: &[u8], num_elements: usize) -> Vec<f32> {
838    const BLOCK_SIZE: usize = 256;
839    const BLOCK_BYTES: usize = 210;
840
841    let num_blocks = (num_elements + BLOCK_SIZE - 1) / BLOCK_SIZE;
842    let mut result = vec![0.0f32; num_blocks * BLOCK_SIZE];
843
844    for sb in 0..num_blocks {
845        let sb_start = sb * BLOCK_BYTES;
846        if sb_start + BLOCK_BYTES > data.len() {
847            break;
848        }
849        let block = &data[sb_start..sb_start + BLOCK_BYTES];
850        let ql = &block[0..128];
851        let qh = &block[128..192];
852        let scales = &block[192..208];
853        let d = f16_to_f32(u16::from_le_bytes([block[208], block[209]]));
854
855        let out_base = sb * BLOCK_SIZE;
856        for group in 0..16usize {
857            let scale = (scales[group] as i8) as f32;
858            let group_off = group * 16;
859            for j in 0..16usize {
860                let idx = group_off + j;
861                let ql_byte = ql[idx / 2];
862                let low4 = if idx % 2 == 0 { ql_byte & 0x0F } else { ql_byte >> 4 };
863                let qh_byte = qh[idx / 4];
864                let high2 = (qh_byte >> ((idx % 4) * 2)) & 0x03;
865                let q6 = ((low4 | (high2 << 4)) as i8).wrapping_sub(32) as f32;
866                result[out_base + idx] = d * scale * q6;
867            }
868        }
869    }
870
871    result.truncate(num_elements);
872    result
873}
874
875/// Dequantize Q5_K to F32.
876///
877/// Q5_K layout per 256-element super-block (176 bytes):
878/// - d[2]:       f16 super-block scale
879/// - dmin[2]:    f16 super-block min scale
880/// - scales[12]: packed 6-bit scales and mins (8 sub-blocks × 2 values)
881/// - qh[32]:     high bit of each 5-bit value (1 bit per element = 32 bytes)
882/// - qs[128]:    lower 4 bits per element (2 per byte)
883///
884/// Value = d * scale * q5 - dmin * min
885fn dequantize_q5k_to_f32(data: &[u8], num_elements: usize) -> Vec<f32> {
886    const BLOCK_SIZE: usize = 256;
887    const BLOCK_BYTES: usize = 176;
888
889    let num_blocks = (num_elements + BLOCK_SIZE - 1) / BLOCK_SIZE;
890    let mut result = vec![0.0f32; num_blocks * BLOCK_SIZE];
891
892    for sb in 0..num_blocks {
893        let sb_start = sb * BLOCK_BYTES;
894        if sb_start + BLOCK_BYTES > data.len() {
895            break;
896        }
897        let block = &data[sb_start..sb_start + BLOCK_BYTES];
898        let d = f16_to_f32(u16::from_le_bytes([block[0], block[1]]));
899        let dmin = f16_to_f32(u16::from_le_bytes([block[2], block[3]]));
900
901        // Unpack 6-bit scales and mins (same layout as Q4K)
902        let sc = &block[4..16];
903        let mut scales = [0u8; 8];
904        let mut mins = [0u8; 8];
905        for i in 0..4 {
906            scales[i] = sc[i] & 0x3F;
907            mins[i] = sc[i + 4] & 0x3F;
908            scales[i + 4] = (sc[i + 8] & 0x0F) | ((sc[i] >> 6) << 4);
909            mins[i + 4] = (sc[i + 8] >> 4) | ((sc[i + 4] >> 6) << 4);
910        }
911
912        let qh = &block[16..48];
913        let qs = &block[48..176];
914
915        let out_base = sb * BLOCK_SIZE;
916        // llama.cpp `dequantize_row_q5_K` layout: process the 256-element
917        // super-block in four 64-element chunks. For each chunk `it`, the low
918        // and high nibble of the SAME `qs` byte land a STRIDE OF 32 apart, and
919        // `qh[l]` holds the 5th bit across the four chunks (u1 = 1<<(2*it),
920        // u2 = 2<<(2*it)). See contracts/q5k-dequant-correctness-v1.yaml.
921        for it in 0..4usize {
922            let ql_base = it * 32;
923            let scale_lo = d * scales[2 * it] as f32;
924            let min_lo = dmin * mins[2 * it] as f32;
925            let scale_hi = d * scales[2 * it + 1] as f32;
926            let min_hi = dmin * mins[2 * it + 1] as f32;
927            let u1 = 1u8 << (2 * it);
928            let u2 = 2u8 << (2 * it);
929            for l in 0..32usize {
930                let qsb = qs[ql_base + l];
931                let qhb = qh[l];
932                let lo = (qsb & 0x0F) + if qhb & u1 != 0 { 16 } else { 0 };
933                result[out_base + it * 64 + l] = scale_lo * (lo as f32) - min_lo;
934                let hi = (qsb >> 4) + if qhb & u2 != 0 { 16 } else { 0 };
935                result[out_base + it * 64 + 32 + l] = scale_hi * (hi as f32) - min_hi;
936            }
937        }
938    }
939
940    result.truncate(num_elements);
941    result
942}
943
944/// Dequantize Q8_0 to F32.
945///
946/// Q8_0 layout per 32-element block (34 bytes):
947/// - d[2]:   f16 block scale
948/// - qs[32]: signed 8-bit quantized values
949///
950/// Value = d * qs[i]
951fn dequantize_q8_0_to_f32(data: &[u8], num_elements: usize) -> Vec<f32> {
952    const BLOCK_SIZE: usize = 32;
953    const BLOCK_BYTES: usize = 34;
954
955    let num_blocks = (num_elements + BLOCK_SIZE - 1) / BLOCK_SIZE;
956    let mut result = vec![0.0f32; num_blocks * BLOCK_SIZE];
957
958    for b in 0..num_blocks {
959        let b_start = b * BLOCK_BYTES;
960        if b_start + BLOCK_BYTES > data.len() {
961            break;
962        }
963        let block = &data[b_start..b_start + BLOCK_BYTES];
964        let d = f16_to_f32(u16::from_le_bytes([block[0], block[1]]));
965        let out_base = b * BLOCK_SIZE;
966        for j in 0..BLOCK_SIZE {
967            result[out_base + j] = d * (block[2 + j] as i8) as f32;
968        }
969    }
970
971    result.truncate(num_elements);
972    result
973}
974
975/// Dequantize Q4_0 to F32.
976///
977/// Q4_0 layout per 32-element block (18 bytes):
978/// - d[2]:   f16 block scale
979/// - qs[16]: 4-bit quantized values, 2 per byte
980///
981/// Value = d * (q4 - 8)  where q4 ∈ 0..15 (centered: subtract 8)
982fn dequantize_q4_0_to_f32(data: &[u8], num_elements: usize) -> Vec<f32> {
983    const BLOCK_SIZE: usize = 32;
984    const BLOCK_BYTES: usize = 18;
985
986    let num_blocks = (num_elements + BLOCK_SIZE - 1) / BLOCK_SIZE;
987    let mut result = vec![0.0f32; num_blocks * BLOCK_SIZE];
988
989    for b in 0..num_blocks {
990        let b_start = b * BLOCK_BYTES;
991        if b_start + BLOCK_BYTES > data.len() {
992            break;
993        }
994        let block = &data[b_start..b_start + BLOCK_BYTES];
995        let d = f16_to_f32(u16::from_le_bytes([block[0], block[1]]));
996        let out_base = b * BLOCK_SIZE;
997        for j in 0..16 {
998            let byte = block[2 + j];
999            let lo = (byte & 0x0F) as i32 - 8;
1000            let hi = ((byte >> 4) & 0x0F) as i32 - 8;
1001            result[out_base + j * 2] = d * lo as f32;
1002            result[out_base + j * 2 + 1] = d * hi as f32;
1003        }
1004    }
1005
1006    result.truncate(num_elements);
1007    result
1008}
1009
1010/// Dequantize Q4_1 to F32.
1011///
1012/// Q4_1 layout per 32-element block (20 bytes):
1013/// - d[2]:   f16 scale
1014/// - m[2]:   f16 min (additive offset)
1015/// - qs[16]: 4-bit quantized values, 2 per byte
1016///
1017/// Value = d * q4 + m  where q4 ∈ 0..15
1018fn dequantize_q4_1_to_f32(data: &[u8], num_elements: usize) -> Vec<f32> {
1019    const BLOCK_SIZE: usize = 32;
1020    const BLOCK_BYTES: usize = 20;
1021
1022    let num_blocks = (num_elements + BLOCK_SIZE - 1) / BLOCK_SIZE;
1023    let mut result = vec![0.0f32; num_blocks * BLOCK_SIZE];
1024
1025    for b in 0..num_blocks {
1026        let b_start = b * BLOCK_BYTES;
1027        if b_start + BLOCK_BYTES > data.len() {
1028            break;
1029        }
1030        let block = &data[b_start..b_start + BLOCK_BYTES];
1031        let d = f16_to_f32(u16::from_le_bytes([block[0], block[1]]));
1032        let m = f16_to_f32(u16::from_le_bytes([block[2], block[3]]));
1033        let out_base = b * BLOCK_SIZE;
1034        for j in 0..16 {
1035            let byte = block[4 + j];
1036            let lo = (byte & 0x0F) as f32;
1037            let hi = ((byte >> 4) & 0x0F) as f32;
1038            result[out_base + j * 2] = d * lo + m;
1039            result[out_base + j * 2 + 1] = d * hi + m;
1040        }
1041    }
1042
1043    result.truncate(num_elements);
1044    result
1045}
1046
1047#[cfg(test)]
1048mod q5k_dequant_tests {
1049    use super::dequantize_q5k_to_f32;
1050
1051    /// Build a single 176-byte Q5_K super-block.
1052    ///
1053    /// `d` / `dmin` are f16 bit patterns. `scale_bytes` is the 12-byte packed
1054    /// scales+mins. `qh` is 32 bytes (one bit per element). `qs` is 128 bytes
1055    /// (two 4-bit values per byte).
1056    fn build_q5k_block(
1057        d: u16,
1058        dmin: u16,
1059        scale_bytes: [u8; 12],
1060        qh: [u8; 32],
1061        qs: [u8; 128],
1062    ) -> Vec<u8> {
1063        let mut block = Vec::with_capacity(176);
1064        block.extend_from_slice(&d.to_le_bytes());
1065        block.extend_from_slice(&dmin.to_le_bytes());
1066        block.extend_from_slice(&scale_bytes);
1067        block.extend_from_slice(&qh);
1068        block.extend_from_slice(&qs);
1069        assert_eq!(block.len(), 176, "Q5_K super-block must be 176 bytes");
1070        block
1071    }
1072
1073    /// Scale bytes that decode (via the Q4_K-style 6-bit unpack) to all eight
1074    /// scales == 1 and all eight mins == 0.
1075    fn scales_all_one_mins_zero() -> [u8; 12] {
1076        // scales[0..4] low6 = bytes[0..4]; mins[0..4] low6 = bytes[4..8];
1077        // scales[4..8] = (bytes[8..12] & 0x0F) | ((bytes[0..4] >> 6) << 4);
1078        // mins[4..8]   = (bytes[8..12] >> 4)   | ((bytes[4..8] >> 6) << 4).
1079        // bytes[0..4]=1 -> scales[0..4]=1, high-2 contribution 0.
1080        // bytes[4..8]=0 -> mins[0..4]=0.
1081        // bytes[8..12]=1 -> scales[4..8]=1, mins[4..8]=0.
1082        [1, 1, 1, 1, 0, 0, 0, 0, 1, 1, 1, 1]
1083    }
1084
1085    /// FALSIFIER (PMAT-842): the GGUF/llama.cpp Q5_K layout places the low and
1086    /// high nibble of the SAME `qs` byte a STRIDE OF 32 apart, NOT at adjacent
1087    /// output indices. With qs[0] = 0x21, low nibble (1) -> out[0], high nibble
1088    /// (2) -> out[32]; out[1] comes from qs[1] (0). The buggy sequential-index
1089    /// implementation produced out[1] == 2.0 and out[32] == 0.0.
1090    #[test]
1091    fn test_q5k_dequant_stride32_nibble_layout() {
1092        let d = 0x3C00; // f16 1.0
1093        let dmin = 0x0000; // f16 0.0
1094        let mut qs = [0u8; 128];
1095        qs[0] = 0x21; // low nibble = 1, high nibble = 2
1096        let block = build_q5k_block(d, dmin, scales_all_one_mins_zero(), [0u8; 32], qs);
1097
1098        let out = dequantize_q5k_to_f32(&block, 256);
1099
1100        // low nibble of qs[0] -> output index 0
1101        assert_eq!(out[0], 1.0, "out[0] must come from qs[0] low nibble (=1)");
1102        // output index 1 comes from qs[1] (=0), NOT qs[0] high nibble
1103        assert_eq!(out[1], 0.0, "out[1] must come from qs[1] low nibble (=0)");
1104        // high nibble of qs[0] -> output index 32 (stride of 32)
1105        assert_eq!(out[32], 2.0, "out[32] must come from qs[0] high nibble (=2)");
1106    }
1107
1108    /// FALSIFIER (PMAT-842): the 5th bit lives in `qh[l]`, with the low nibble
1109    /// using bit (2*it) and the high nibble using bit (2*it+1). For the first
1110    /// chunk (it=0): u1 = 0b01, u2 = 0b10. With qh[0] = 0x03 both bits are set,
1111    /// so the low value gains +16 (out[0]=17) and the high value gains +16
1112    /// (out[32]=18).
1113    #[test]
1114    fn test_q5k_dequant_fifth_bit_placement() {
1115        let d = 0x3C00; // f16 1.0
1116        let dmin = 0x0000; // f16 0.0
1117        let mut qs = [0u8; 128];
1118        qs[0] = 0x21; // low nibble = 1, high nibble = 2
1119        let mut qh = [0u8; 32];
1120        qh[0] = 0x03; // set bits 0 (u1) and 1 (u2) for chunk it=0
1121        let block = build_q5k_block(d, dmin, scales_all_one_mins_zero(), qh, qs);
1122
1123        let out = dequantize_q5k_to_f32(&block, 256);
1124
1125        // low nibble: 1 + 16 (bit 0 set) = 17
1126        assert_eq!(out[0], 17.0, "out[0] low nibble must add 16 for qh bit 0");
1127        // high nibble: 2 + 16 (bit 1 set) = 18, at stride-32 index
1128        assert_eq!(out[32], 18.0, "out[32] high nibble must add 16 for qh bit 1");
1129    }
1130}