1use crate::backends::q4k::matmul_q4k_f32_dispatch;
7use crate::blis::attention::fused_attention_decode;
8use crate::blis::norms::rms_norm;
9use crate::error::TruenoError;
10use crate::inference::gguf::{GgmlType, GgufFile};
11
12#[derive(Debug, Clone)]
14pub struct ModelConfig {
15 pub hidden_size: usize,
16 pub intermediate_size: usize,
17 pub num_layers: usize,
18 pub num_heads: usize,
19 pub num_kv_heads: usize,
20 pub head_dim: usize,
21 pub vocab_size: usize,
22 pub rms_norm_eps: f32,
23 pub rope_theta: f32,
24 pub max_seq_len: usize,
25 pub arch: String,
26}
27
28impl ModelConfig {
29 pub fn from_gguf(gguf: &GgufFile) -> Result<Self, TruenoError> {
31 let arch = gguf.meta_str("general.architecture").unwrap_or("llama").to_string();
32 let prefix = &arch; let hidden_size = gguf
35 .meta_u32(&format!("{prefix}.embedding_length"))
36 .ok_or_else(|| TruenoError::InvalidInput("Missing embedding_length in GGUF".into()))?
37 as usize;
38
39 let num_heads = gguf
40 .meta_u32(&format!("{prefix}.attention.head_count"))
41 .ok_or_else(|| TruenoError::InvalidInput("Missing head_count in GGUF".into()))?
42 as usize;
43
44 let num_kv_heads = gguf
45 .meta_u32(&format!("{prefix}.attention.head_count_kv"))
46 .unwrap_or(num_heads as u32) as usize;
47
48 let num_layers = gguf
49 .meta_u32(&format!("{prefix}.block_count"))
50 .ok_or_else(|| TruenoError::InvalidInput("Missing block_count in GGUF".into()))?
51 as usize;
52
53 let intermediate_size =
54 gguf.meta_u32(&format!("{prefix}.feed_forward_length")).ok_or_else(|| {
55 TruenoError::InvalidInput("Missing feed_forward_length in GGUF".into())
56 })? as usize;
57
58 let head_dim = hidden_size / num_heads;
59
60 let vocab_size = gguf
61 .meta_u32("tokenizer.ggml.vocab_size")
62 .or_else(|| {
63 gguf.metadata.get("tokenizer.ggml.tokens").and_then(|v| {
65 if let crate::inference::gguf::MetadataValue::Array(arr) = v {
66 Some(arr.len() as u32)
67 } else {
68 None
69 }
70 })
71 })
72 .unwrap_or(32000) as usize;
73
74 let rms_norm_eps =
75 gguf.meta_f32(&format!("{prefix}.attention.layer_norm_rms_epsilon")).unwrap_or(1e-5);
76
77 let rope_theta = gguf.meta_f32(&format!("{prefix}.rope.freq_base")).unwrap_or(10000.0);
78
79 let max_seq_len =
80 gguf.meta_u32(&format!("{prefix}.context_length")).unwrap_or(2048) as usize;
81
82 Ok(Self {
83 hidden_size,
84 intermediate_size,
85 num_layers,
86 num_heads,
87 num_kv_heads,
88 head_dim,
89 vocab_size,
90 rms_norm_eps,
91 rope_theta,
92 max_seq_len,
93 arch,
94 })
95 }
96}
97
98pub enum WeightMatrix {
100 Q4K { data: Vec<u8>, rows: usize },
102 F32 { data: Vec<f32>, rows: usize },
104}
105
106impl WeightMatrix {
107 pub fn rows(&self) -> usize {
108 match self {
109 WeightMatrix::Q4K { rows, .. } => *rows,
110 WeightMatrix::F32 { rows, .. } => *rows,
111 }
112 }
113}
114
115pub struct LayerWeights {
117 pub attn_norm: Vec<f32>,
119 pub q_weight: WeightMatrix,
120 pub k_weight: WeightMatrix,
121 pub v_weight: WeightMatrix,
122 pub o_weight: WeightMatrix,
123 pub q_bias: Option<Vec<f32>>,
125 pub k_bias: Option<Vec<f32>>,
126 pub v_bias: Option<Vec<f32>>,
127
128 pub ffn_norm: Vec<f32>,
130 pub gate_weight: WeightMatrix,
131 pub up_weight: WeightMatrix,
132 pub down_weight: WeightMatrix,
133}
134
135pub struct ModelWeights {
137 pub token_embd: Vec<f32>, pub output_norm: Vec<f32>, pub output_weight: WeightMatrix,
140 pub layers: Vec<LayerWeights>,
141}
142
143pub struct ForwardArena {
147 pub attn_input: Vec<f32>,
148 pub q: Vec<f32>,
149 pub k: Vec<f32>,
150 pub v: Vec<f32>,
151 pub attn_out: Vec<f32>,
152 pub attn_proj: Vec<f32>,
153 pub ffn_input: Vec<f32>,
154 pub gate: Vec<f32>,
155 pub up: Vec<f32>,
156 pub swiglu: Vec<f32>,
157 pub ffn_out: Vec<f32>,
158 pub hidden: Vec<f32>,
159 pub residual: Vec<f32>,
160 pub normed: Vec<f32>,
161 pub k_cache_head: Vec<f32>,
162 pub v_cache_head: Vec<f32>,
163}
164
165impl ForwardArena {
166 pub fn new(config: &ModelConfig) -> Self {
167 let kv_dim = config.num_kv_heads * config.head_dim;
168 let head_cache_size = config.max_seq_len * config.head_dim;
169 Self {
170 attn_input: vec![0.0f32; config.hidden_size],
171 q: vec![0.0f32; config.hidden_size],
172 k: vec![0.0f32; kv_dim],
173 v: vec![0.0f32; kv_dim],
174 attn_out: vec![0.0f32; config.hidden_size],
175 attn_proj: vec![0.0f32; config.hidden_size],
176 ffn_input: vec![0.0f32; config.hidden_size],
177 gate: vec![0.0f32; config.intermediate_size],
178 up: vec![0.0f32; config.intermediate_size],
179 swiglu: vec![0.0f32; config.intermediate_size],
180 ffn_out: vec![0.0f32; config.hidden_size],
181 hidden: vec![0.0f32; config.hidden_size],
182 residual: vec![0.0f32; config.hidden_size],
183 normed: vec![0.0f32; config.hidden_size],
184 k_cache_head: vec![0.0f32; head_cache_size],
185 v_cache_head: vec![0.0f32; head_cache_size],
186 }
187 }
188}
189
190pub struct KvCache {
192 pub k: Vec<Vec<f32>>,
194 pub v: Vec<Vec<f32>>,
196 pub seq_len: usize,
197}
198
199impl KvCache {
200 pub fn new(config: &ModelConfig) -> Self {
201 let kv_dim = config.num_kv_heads * config.head_dim;
202 let layer_size = config.max_seq_len * kv_dim;
203 Self {
204 k: (0..config.num_layers).map(|_| vec![0.0f32; layer_size]).collect(),
205 v: (0..config.num_layers).map(|_| vec![0.0f32; layer_size]).collect(),
206 seq_len: 0,
207 }
208 }
209}
210
211pub struct LlamaModel {
213 pub config: ModelConfig,
214 pub weights: ModelWeights,
215}
216
217impl LlamaModel {
218 pub fn from_gguf(gguf: &GgufFile) -> Result<Self, TruenoError> {
220 let config = ModelConfig::from_gguf(gguf)?;
221
222 eprintln!(
223 "Loading {} model: {}L × {}H ({}h {}kv) × {}I, vocab={}",
224 config.arch,
225 config.num_layers,
226 config.hidden_size,
227 config.num_heads,
228 config.num_kv_heads,
229 config.intermediate_size,
230 config.vocab_size,
231 );
232
233 let weights = load_weights(gguf, &config)?;
234
235 Ok(Self { config, weights })
236 }
237
238 pub fn forward(
242 &self,
243 token_id: u32,
244 pos: usize,
245 kv_cache: &mut KvCache,
246 arena: &mut ForwardArena,
247 ) -> Result<Vec<f32>, TruenoError> {
248 let cfg = &self.config;
249 let w = &self.weights;
250
251 let embd_start = token_id as usize * cfg.hidden_size;
253 let embd_end = embd_start + cfg.hidden_size;
254 if embd_end > w.token_embd.len() {
255 return Err(TruenoError::InvalidInput(format!(
256 "Token ID {token_id} out of range (vocab={})",
257 cfg.vocab_size
258 )));
259 }
260 arena.hidden[..cfg.hidden_size].copy_from_slice(&w.token_embd[embd_start..embd_end]);
261
262 for (layer_idx, lw) in w.layers.iter().enumerate() {
264 self.forward_layer(layer_idx, lw, pos, kv_cache, arena)?;
265 }
266
267 rms_norm(
269 &arena.hidden[..cfg.hidden_size],
270 &w.output_norm,
271 cfg.rms_norm_eps,
272 &mut arena.normed[..cfg.hidden_size],
273 )?;
274
275 let logits =
277 matmul_weight(&w.output_weight, &arena.normed[..cfg.hidden_size], cfg.hidden_size);
278
279 Ok(logits)
280 }
281
282 fn forward_layer(
284 &self,
285 layer_idx: usize,
286 lw: &LayerWeights,
287 pos: usize,
288 kv_cache: &mut KvCache,
289 arena: &mut ForwardArena,
290 ) -> Result<(), TruenoError> {
291 let cfg = &self.config;
292 let kv_dim = cfg.num_kv_heads * cfg.head_dim;
293 let h_sz = cfg.hidden_size;
294
295 rms_norm(
297 &arena.hidden[..h_sz],
298 &lw.attn_norm,
299 cfg.rms_norm_eps,
300 &mut arena.attn_input[..h_sz],
301 )?;
302
303 matmul_weight_into(&lw.q_weight, &arena.attn_input[..h_sz], h_sz, &mut arena.q[..h_sz]);
305 matmul_weight_into(&lw.k_weight, &arena.attn_input[..h_sz], h_sz, &mut arena.k[..kv_dim]);
306 matmul_weight_into(&lw.v_weight, &arena.attn_input[..h_sz], h_sz, &mut arena.v[..kv_dim]);
307
308 if let Some(bias) = &lw.q_bias {
310 for (v, b) in arena.q[..h_sz].iter_mut().zip(bias.iter()) {
311 *v += b;
312 }
313 }
314 if let Some(bias) = &lw.k_bias {
315 for (v, b) in arena.k[..kv_dim].iter_mut().zip(bias.iter()) {
316 *v += b;
317 }
318 }
319 if let Some(bias) = &lw.v_bias {
320 for (v, b) in arena.v[..kv_dim].iter_mut().zip(bias.iter()) {
321 *v += b;
322 }
323 }
324
325 apply_rope(&mut arena.q[..h_sz], cfg.num_heads, cfg.head_dim, pos, cfg.rope_theta);
327 apply_rope(&mut arena.k[..kv_dim], cfg.num_kv_heads, cfg.head_dim, pos, cfg.rope_theta);
328
329 let kv_off = pos * kv_dim;
331 kv_cache.k[layer_idx][kv_off..kv_off + kv_dim].copy_from_slice(&arena.k[..kv_dim]);
332 kv_cache.v[layer_idx][kv_off..kv_off + kv_dim].copy_from_slice(&arena.v[..kv_dim]);
333
334 let seq_len = pos + 1;
335
336 arena.attn_out[..h_sz].fill(0.0);
338 let heads_per_kv = cfg.num_heads / cfg.num_kv_heads;
339
340 for h in 0..cfg.num_heads {
341 let kv_h = h / heads_per_kv;
342 let q_head = &arena.q[h * cfg.head_dim..(h + 1) * cfg.head_dim];
343
344 let view_len = seq_len * cfg.head_dim;
346 for s in 0..seq_len {
347 let src_off = s * kv_dim + kv_h * cfg.head_dim;
348 let dst_off = s * cfg.head_dim;
349 arena.k_cache_head[dst_off..dst_off + cfg.head_dim]
350 .copy_from_slice(&kv_cache.k[layer_idx][src_off..src_off + cfg.head_dim]);
351 arena.v_cache_head[dst_off..dst_off + cfg.head_dim]
352 .copy_from_slice(&kv_cache.v[layer_idx][src_off..src_off + cfg.head_dim]);
353 }
354
355 let out_head = &mut arena.attn_out[h * cfg.head_dim..(h + 1) * cfg.head_dim];
356 fused_attention_decode(
357 q_head,
358 &arena.k_cache_head[..view_len],
359 &arena.v_cache_head[..view_len],
360 cfg.head_dim,
361 seq_len,
362 out_head,
363 );
364 }
365
366 matmul_weight_into(
368 &lw.o_weight,
369 &arena.attn_out[..h_sz],
370 h_sz,
371 &mut arena.attn_proj[..h_sz],
372 );
373
374 for i in 0..h_sz {
376 arena.residual[i] = arena.hidden[i] + arena.attn_proj[i];
377 }
378
379 rms_norm(
381 &arena.residual[..h_sz],
382 &lw.ffn_norm,
383 cfg.rms_norm_eps,
384 &mut arena.ffn_input[..h_sz],
385 )?;
386
387 let i_sz = cfg.intermediate_size;
388 matmul_weight_into(
389 &lw.gate_weight,
390 &arena.ffn_input[..h_sz],
391 h_sz,
392 &mut arena.gate[..i_sz],
393 );
394 matmul_weight_into(&lw.up_weight, &arena.ffn_input[..h_sz], h_sz, &mut arena.up[..i_sz]);
395
396 for i in 0..i_sz {
398 let g = arena.gate[i];
399 let silu_g = g / (1.0 + (-g).exp());
400 arena.swiglu[i] = silu_g * arena.up[i];
401 }
402
403 matmul_weight_into(
405 &lw.down_weight,
406 &arena.swiglu[..i_sz],
407 i_sz,
408 &mut arena.ffn_out[..h_sz],
409 );
410
411 for i in 0..h_sz {
413 arena.hidden[i] = arena.residual[i] + arena.ffn_out[i];
414 }
415
416 Ok(())
417 }
418}
419
420fn apply_rope(x: &mut [f32], num_heads: usize, head_dim: usize, pos: usize, theta: f32) {
422 for h in 0..num_heads {
423 let head = &mut x[h * head_dim..(h + 1) * head_dim];
424 for i in (0..head_dim).step_by(2) {
425 let freq = 1.0 / theta.powf(i as f32 / head_dim as f32);
426 let angle = pos as f32 * freq;
427 let (sin_a, cos_a) = angle.sin_cos();
428 let x0 = head[i];
429 let x1 = head[i + 1];
430 head[i] = x0 * cos_a - x1 * sin_a;
431 head[i + 1] = x0 * sin_a + x1 * cos_a;
432 }
433 }
434}
435
436fn load_weights(gguf: &GgufFile, config: &ModelConfig) -> Result<ModelWeights, TruenoError> {
438 let token_embd = load_f32_or_dequant_tensor(
442 gguf,
443 "token_embd.weight",
444 config.vocab_size * config.hidden_size,
445 )?;
446
447 let output_norm = load_f32_tensor(gguf, "output_norm.weight", config.hidden_size)?;
449
450 let output_weight = if gguf.tensor_info("output.weight").is_some() {
453 load_weight_matrix(gguf, "output.weight", config.hidden_size)?
454 } else {
455 WeightMatrix::F32 { data: token_embd.clone(), rows: config.vocab_size }
457 };
458
459 let mut layers = Vec::with_capacity(config.num_layers);
461 for i in 0..config.num_layers {
462 let prefix = format!("blk.{i}");
463
464 let attn_norm =
465 load_f32_tensor(gguf, &format!("{prefix}.attn_norm.weight"), config.hidden_size)?;
466 let ffn_norm =
467 load_f32_tensor(gguf, &format!("{prefix}.ffn_norm.weight"), config.hidden_size)?;
468
469 let q_weight =
470 load_weight_matrix(gguf, &format!("{prefix}.attn_q.weight"), config.hidden_size)?;
471 let k_weight =
472 load_weight_matrix(gguf, &format!("{prefix}.attn_k.weight"), config.hidden_size)?;
473 let v_weight =
474 load_weight_matrix(gguf, &format!("{prefix}.attn_v.weight"), config.hidden_size)?;
475 let o_weight =
476 load_weight_matrix(gguf, &format!("{prefix}.attn_output.weight"), config.hidden_size)?;
477
478 let kv_dim = config.num_kv_heads * config.head_dim;
480 let q_bias = load_optional_f32(gguf, &format!("{prefix}.attn_q.bias"), config.hidden_size);
481 let k_bias = load_optional_f32(gguf, &format!("{prefix}.attn_k.bias"), kv_dim);
482 let v_bias = load_optional_f32(gguf, &format!("{prefix}.attn_v.bias"), kv_dim);
483
484 let gate_weight =
485 load_weight_matrix(gguf, &format!("{prefix}.ffn_gate.weight"), config.hidden_size)?;
486 let up_weight =
487 load_weight_matrix(gguf, &format!("{prefix}.ffn_up.weight"), config.hidden_size)?;
488 let down_weight = load_weight_matrix(
489 gguf,
490 &format!("{prefix}.ffn_down.weight"),
491 config.intermediate_size,
492 )?;
493
494 if i == 0 {
495 eprintln!(
496 " Layer 0: Q[{}×{}] K[{}×{}] V[{}×{}] Gate[{}×{}]",
497 q_weight.rows(),
498 config.hidden_size,
499 k_weight.rows(),
500 config.hidden_size,
501 v_weight.rows(),
502 config.hidden_size,
503 gate_weight.rows(),
504 config.hidden_size,
505 );
506 }
507
508 layers.push(LayerWeights {
509 attn_norm,
510 q_weight,
511 k_weight,
512 v_weight,
513 o_weight,
514 q_bias,
515 k_bias,
516 v_bias,
517 ffn_norm,
518 gate_weight,
519 up_weight,
520 down_weight,
521 });
522 }
523
524 eprintln!(" Loaded {} layers", layers.len());
525
526 Ok(ModelWeights { token_embd, output_norm, output_weight, layers })
527}
528
529#[deny(clippy::wildcard_enum_match_arm)]
532fn load_f32_or_dequant_tensor(
533 gguf: &GgufFile,
534 name: &str,
535 expected_elements: usize,
536) -> Result<Vec<f32>, TruenoError> {
537 let info = gguf
538 .tensor_info(name)
539 .ok_or_else(|| TruenoError::InvalidInput(format!("Missing tensor: {name}")))?;
540 let data = gguf
541 .tensor_data(name)
542 .ok_or_else(|| TruenoError::InvalidInput(format!("Missing tensor data: {name}")))?;
543
544 match info.dtype {
545 GgmlType::F32 | GgmlType::F16 | GgmlType::BF16 => {
546 Ok(to_f32_from_any(data, info.dtype, expected_elements))
547 }
548 GgmlType::Q4K => {
549 let n_elements = info.n_elements() as usize;
550 Ok(crate::backends::q4k::dequantize_q4k_to_f32(data, n_elements))
551 }
552 GgmlType::Q6K => Ok(dequantize_q6k_to_f32(data, info.n_elements() as usize)),
553 GgmlType::Q5K => Ok(dequantize_q5k_to_f32(data, info.n_elements() as usize)),
554 GgmlType::Q8_0 => Ok(dequantize_q8_0_to_f32(data, info.n_elements() as usize)),
555 GgmlType::Q4_0 => Ok(dequantize_q4_0_to_f32(data, info.n_elements() as usize)),
556 GgmlType::Q4_1 => Ok(dequantize_q4_1_to_f32(data, info.n_elements() as usize)),
557 GgmlType::Q5_0
559 | GgmlType::Q5_1
560 | GgmlType::Q8_1
561 | GgmlType::Q2K
562 | GgmlType::Q3K
563 | GgmlType::Q8K
564 | GgmlType::IQ2XXS
565 | GgmlType::IQ2XS
566 | GgmlType::IQ3XXS
567 | GgmlType::IQ1S
568 | GgmlType::IQ4NL
569 | GgmlType::IQ3S
570 | GgmlType::IQ2S
571 | GgmlType::IQ4XS
572 | GgmlType::I8
573 | GgmlType::I16
574 | GgmlType::I32
575 | GgmlType::I64
576 | GgmlType::F64
577 | GgmlType::IQ1M
578 | GgmlType::TQ1_0
579 | GgmlType::TQ2_0
580 | GgmlType::MXFP4
581 | GgmlType::NVFP4
582 | GgmlType::Q1_0
583 | GgmlType::Q2_0 => {
584 eprintln!(
585 " WARNING: tensor '{name}' has unsupported dtype {:?}, using zeros",
586 info.dtype
587 );
588 Ok(vec![0.0f32; expected_elements])
589 }
590 }
591}
592
593fn load_optional_f32(gguf: &GgufFile, name: &str, expected_elements: usize) -> Option<Vec<f32>> {
595 let info = gguf.tensor_info(name)?;
596 let data = gguf.tensor_data(name)?;
597 Some(to_f32_from_any(data, info.dtype, expected_elements))
598}
599
600fn load_f32_tensor(
602 gguf: &GgufFile,
603 name: &str,
604 expected_elements: usize,
605) -> Result<Vec<f32>, TruenoError> {
606 let info = gguf
607 .tensor_info(name)
608 .ok_or_else(|| TruenoError::InvalidInput(format!("Missing tensor: {name}")))?;
609 let data = gguf
610 .tensor_data(name)
611 .ok_or_else(|| TruenoError::InvalidInput(format!("Missing tensor data: {name}")))?;
612
613 Ok(to_f32_from_any(data, info.dtype, expected_elements))
614}
615
616#[deny(clippy::wildcard_enum_match_arm)]
620fn load_weight_matrix(
621 gguf: &GgufFile,
622 name: &str,
623 in_dim: usize,
624) -> Result<WeightMatrix, TruenoError> {
625 let info = gguf
626 .tensor_info(name)
627 .ok_or_else(|| TruenoError::InvalidInput(format!("Missing tensor: {name}")))?;
628 let data = gguf
629 .tensor_data(name)
630 .ok_or_else(|| TruenoError::InvalidInput(format!("Missing tensor data: {name}")))?;
631
632 let n_elements = info.n_elements() as usize;
633 let out_dim = n_elements / in_dim;
634
635 match info.dtype {
636 GgmlType::Q4K => Ok(WeightMatrix::Q4K { data: data.to_vec(), rows: out_dim }),
637 GgmlType::F32 | GgmlType::F16 | GgmlType::BF16 => {
638 let f32_data = to_f32_from_any(data, info.dtype, n_elements);
639 Ok(WeightMatrix::F32 { data: f32_data, rows: out_dim })
640 }
641 GgmlType::Q6K => {
642 let f32_data = dequantize_q6k_to_f32(data, n_elements);
643 Ok(WeightMatrix::F32 { data: f32_data, rows: out_dim })
644 }
645 GgmlType::Q5K => {
646 let f32_data = dequantize_q5k_to_f32(data, n_elements);
647 Ok(WeightMatrix::F32 { data: f32_data, rows: out_dim })
648 }
649 GgmlType::Q8_0 => {
650 let f32_data = dequantize_q8_0_to_f32(data, n_elements);
651 Ok(WeightMatrix::F32 { data: f32_data, rows: out_dim })
652 }
653 GgmlType::Q4_0 => {
654 let f32_data = dequantize_q4_0_to_f32(data, n_elements);
655 Ok(WeightMatrix::F32 { data: f32_data, rows: out_dim })
656 }
657 GgmlType::Q4_1 => {
658 let f32_data = dequantize_q4_1_to_f32(data, n_elements);
659 Ok(WeightMatrix::F32 { data: f32_data, rows: out_dim })
660 }
661 GgmlType::Q5_0
663 | GgmlType::Q5_1
664 | GgmlType::Q8_1
665 | GgmlType::Q2K
666 | GgmlType::Q3K
667 | GgmlType::Q8K
668 | GgmlType::IQ2XXS
669 | GgmlType::IQ2XS
670 | GgmlType::IQ3XXS
671 | GgmlType::IQ1S
672 | GgmlType::IQ4NL
673 | GgmlType::IQ3S
674 | GgmlType::IQ2S
675 | GgmlType::IQ4XS
676 | GgmlType::I8
677 | GgmlType::I16
678 | GgmlType::I32
679 | GgmlType::I64
680 | GgmlType::F64
681 | GgmlType::IQ1M
682 | GgmlType::TQ1_0
683 | GgmlType::TQ2_0
684 | GgmlType::MXFP4
685 | GgmlType::NVFP4
686 | GgmlType::Q1_0
687 | GgmlType::Q2_0 => {
688 eprintln!(" WARNING: tensor '{name}' dtype {:?} unsupported, using zeros", info.dtype);
689 Ok(WeightMatrix::F32 { data: vec![0.0f32; n_elements], rows: out_dim })
690 }
691 }
692}
693
694fn matmul_weight(weight: &WeightMatrix, input: &[f32], in_dim: usize) -> Vec<f32> {
697 match weight {
698 WeightMatrix::Q4K { data, rows } => matmul_q4k_f32_dispatch(data, input, *rows, in_dim),
699 WeightMatrix::F32 { data, rows } => {
700 let mut out = vec![0.0f32; *rows];
701 for i in 0..*rows {
702 let row = &data[i * in_dim..(i + 1) * in_dim];
703 out[i] = row.iter().zip(input.iter()).map(|(a, b)| a * b).sum();
704 }
705 out
706 }
707 }
708}
709
710fn matmul_weight_into(weight: &WeightMatrix, input: &[f32], in_dim: usize, out: &mut [f32]) {
712 match weight {
713 WeightMatrix::Q4K { data, rows } => {
714 let result = matmul_q4k_f32_dispatch(data, input, *rows, in_dim);
715 out[..*rows].copy_from_slice(&result);
716 }
717 WeightMatrix::F32 { data, rows } => {
718 for i in 0..*rows {
719 let row = &data[i * in_dim..(i + 1) * in_dim];
720 out[i] = row.iter().zip(input.iter()).map(|(a, b)| a * b).sum();
721 }
722 }
723 }
724}
725
726fn f16_to_f32(bits: u16) -> f32 {
728 let sign = ((bits >> 15) as u32) << 31;
729 let exp = ((bits >> 10) & 0x1F) as u32;
730 let mant = (bits & 0x3FF) as u32;
731
732 if exp == 0 {
733 if mant == 0 {
734 return f32::from_bits(sign); }
736 let mut m = mant;
738 let mut e: i32 = -14;
739 while m & 0x400 == 0 {
740 m <<= 1;
741 e -= 1;
742 }
743 m &= 0x3FF;
744 let f32_exp = ((e + 127) as u32) << 23;
745 return f32::from_bits(sign | f32_exp | (m << 13));
746 }
747 if exp == 31 {
748 return f32::from_bits(sign | 0x7F80_0000 | (mant << 13));
750 }
751 let f32_exp = (exp + 112) << 23; f32::from_bits(sign | f32_exp | (mant << 13))
753}
754
755#[deny(clippy::wildcard_enum_match_arm)]
757fn to_f32_from_any(data: &[u8], dtype: GgmlType, n_elements: usize) -> Vec<f32> {
758 match dtype {
759 GgmlType::F32 => {
760 let count = n_elements.min(data.len() / 4);
762 (0..count)
763 .map(|i| {
764 let off = i * 4;
765 f32::from_le_bytes([data[off], data[off + 1], data[off + 2], data[off + 3]])
766 })
767 .collect()
768 }
769 GgmlType::F16 => {
770 let count = n_elements.min(data.len() / 2);
771 (0..count)
772 .map(|i| {
773 let off = i * 2;
774 let bits = u16::from_le_bytes([data[off], data[off + 1]]);
775 f16_to_f32(bits)
776 })
777 .collect()
778 }
779 GgmlType::BF16 => {
780 let count = n_elements.min(data.len() / 2);
781 (0..count)
782 .map(|i| {
783 let off = i * 2;
784 let bits = u16::from_le_bytes([data[off], data[off + 1]]);
785 f32::from_bits((bits as u32) << 16)
786 })
787 .collect()
788 }
789 GgmlType::Q4_0
791 | GgmlType::Q4_1
792 | GgmlType::Q5_0
793 | GgmlType::Q5_1
794 | GgmlType::Q8_0
795 | GgmlType::Q8_1
796 | GgmlType::Q2K
797 | GgmlType::Q3K
798 | GgmlType::Q4K
799 | GgmlType::Q5K
800 | GgmlType::Q6K
801 | GgmlType::Q8K
802 | GgmlType::IQ2XXS
803 | GgmlType::IQ2XS
804 | GgmlType::IQ3XXS
805 | GgmlType::IQ1S
806 | GgmlType::IQ4NL
807 | GgmlType::IQ3S
808 | GgmlType::IQ2S
809 | GgmlType::IQ4XS
810 | GgmlType::I8
811 | GgmlType::I16
812 | GgmlType::I32
813 | GgmlType::I64
814 | GgmlType::F64
815 | GgmlType::IQ1M
816 | GgmlType::TQ1_0
817 | GgmlType::TQ2_0
818 | GgmlType::MXFP4
819 | GgmlType::NVFP4
820 | GgmlType::Q1_0
821 | GgmlType::Q2_0 => {
822 vec![0.0f32; n_elements]
824 }
825 }
826}
827
828fn dequantize_q6k_to_f32(data: &[u8], num_elements: usize) -> Vec<f32> {
838 const BLOCK_SIZE: usize = 256;
839 const BLOCK_BYTES: usize = 210;
840
841 let num_blocks = (num_elements + BLOCK_SIZE - 1) / BLOCK_SIZE;
842 let mut result = vec![0.0f32; num_blocks * BLOCK_SIZE];
843
844 for sb in 0..num_blocks {
845 let sb_start = sb * BLOCK_BYTES;
846 if sb_start + BLOCK_BYTES > data.len() {
847 break;
848 }
849 let block = &data[sb_start..sb_start + BLOCK_BYTES];
850 let ql = &block[0..128];
851 let qh = &block[128..192];
852 let scales = &block[192..208];
853 let d = f16_to_f32(u16::from_le_bytes([block[208], block[209]]));
854
855 let out_base = sb * BLOCK_SIZE;
856 for group in 0..16usize {
857 let scale = (scales[group] as i8) as f32;
858 let group_off = group * 16;
859 for j in 0..16usize {
860 let idx = group_off + j;
861 let ql_byte = ql[idx / 2];
862 let low4 = if idx % 2 == 0 { ql_byte & 0x0F } else { ql_byte >> 4 };
863 let qh_byte = qh[idx / 4];
864 let high2 = (qh_byte >> ((idx % 4) * 2)) & 0x03;
865 let q6 = ((low4 | (high2 << 4)) as i8).wrapping_sub(32) as f32;
866 result[out_base + idx] = d * scale * q6;
867 }
868 }
869 }
870
871 result.truncate(num_elements);
872 result
873}
874
875fn dequantize_q5k_to_f32(data: &[u8], num_elements: usize) -> Vec<f32> {
886 const BLOCK_SIZE: usize = 256;
887 const BLOCK_BYTES: usize = 176;
888
889 let num_blocks = (num_elements + BLOCK_SIZE - 1) / BLOCK_SIZE;
890 let mut result = vec![0.0f32; num_blocks * BLOCK_SIZE];
891
892 for sb in 0..num_blocks {
893 let sb_start = sb * BLOCK_BYTES;
894 if sb_start + BLOCK_BYTES > data.len() {
895 break;
896 }
897 let block = &data[sb_start..sb_start + BLOCK_BYTES];
898 let d = f16_to_f32(u16::from_le_bytes([block[0], block[1]]));
899 let dmin = f16_to_f32(u16::from_le_bytes([block[2], block[3]]));
900
901 let sc = &block[4..16];
903 let mut scales = [0u8; 8];
904 let mut mins = [0u8; 8];
905 for i in 0..4 {
906 scales[i] = sc[i] & 0x3F;
907 mins[i] = sc[i + 4] & 0x3F;
908 scales[i + 4] = (sc[i + 8] & 0x0F) | ((sc[i] >> 6) << 4);
909 mins[i + 4] = (sc[i + 8] >> 4) | ((sc[i + 4] >> 6) << 4);
910 }
911
912 let qh = &block[16..48];
913 let qs = &block[48..176];
914
915 let out_base = sb * BLOCK_SIZE;
916 for it in 0..4usize {
922 let ql_base = it * 32;
923 let scale_lo = d * scales[2 * it] as f32;
924 let min_lo = dmin * mins[2 * it] as f32;
925 let scale_hi = d * scales[2 * it + 1] as f32;
926 let min_hi = dmin * mins[2 * it + 1] as f32;
927 let u1 = 1u8 << (2 * it);
928 let u2 = 2u8 << (2 * it);
929 for l in 0..32usize {
930 let qsb = qs[ql_base + l];
931 let qhb = qh[l];
932 let lo = (qsb & 0x0F) + if qhb & u1 != 0 { 16 } else { 0 };
933 result[out_base + it * 64 + l] = scale_lo * (lo as f32) - min_lo;
934 let hi = (qsb >> 4) + if qhb & u2 != 0 { 16 } else { 0 };
935 result[out_base + it * 64 + 32 + l] = scale_hi * (hi as f32) - min_hi;
936 }
937 }
938 }
939
940 result.truncate(num_elements);
941 result
942}
943
944fn dequantize_q8_0_to_f32(data: &[u8], num_elements: usize) -> Vec<f32> {
952 const BLOCK_SIZE: usize = 32;
953 const BLOCK_BYTES: usize = 34;
954
955 let num_blocks = (num_elements + BLOCK_SIZE - 1) / BLOCK_SIZE;
956 let mut result = vec![0.0f32; num_blocks * BLOCK_SIZE];
957
958 for b in 0..num_blocks {
959 let b_start = b * BLOCK_BYTES;
960 if b_start + BLOCK_BYTES > data.len() {
961 break;
962 }
963 let block = &data[b_start..b_start + BLOCK_BYTES];
964 let d = f16_to_f32(u16::from_le_bytes([block[0], block[1]]));
965 let out_base = b * BLOCK_SIZE;
966 for j in 0..BLOCK_SIZE {
967 result[out_base + j] = d * (block[2 + j] as i8) as f32;
968 }
969 }
970
971 result.truncate(num_elements);
972 result
973}
974
975fn dequantize_q4_0_to_f32(data: &[u8], num_elements: usize) -> Vec<f32> {
983 const BLOCK_SIZE: usize = 32;
984 const BLOCK_BYTES: usize = 18;
985
986 let num_blocks = (num_elements + BLOCK_SIZE - 1) / BLOCK_SIZE;
987 let mut result = vec![0.0f32; num_blocks * BLOCK_SIZE];
988
989 for b in 0..num_blocks {
990 let b_start = b * BLOCK_BYTES;
991 if b_start + BLOCK_BYTES > data.len() {
992 break;
993 }
994 let block = &data[b_start..b_start + BLOCK_BYTES];
995 let d = f16_to_f32(u16::from_le_bytes([block[0], block[1]]));
996 let out_base = b * BLOCK_SIZE;
997 for j in 0..16 {
998 let byte = block[2 + j];
999 let lo = (byte & 0x0F) as i32 - 8;
1000 let hi = ((byte >> 4) & 0x0F) as i32 - 8;
1001 result[out_base + j * 2] = d * lo as f32;
1002 result[out_base + j * 2 + 1] = d * hi as f32;
1003 }
1004 }
1005
1006 result.truncate(num_elements);
1007 result
1008}
1009
1010fn dequantize_q4_1_to_f32(data: &[u8], num_elements: usize) -> Vec<f32> {
1019 const BLOCK_SIZE: usize = 32;
1020 const BLOCK_BYTES: usize = 20;
1021
1022 let num_blocks = (num_elements + BLOCK_SIZE - 1) / BLOCK_SIZE;
1023 let mut result = vec![0.0f32; num_blocks * BLOCK_SIZE];
1024
1025 for b in 0..num_blocks {
1026 let b_start = b * BLOCK_BYTES;
1027 if b_start + BLOCK_BYTES > data.len() {
1028 break;
1029 }
1030 let block = &data[b_start..b_start + BLOCK_BYTES];
1031 let d = f16_to_f32(u16::from_le_bytes([block[0], block[1]]));
1032 let m = f16_to_f32(u16::from_le_bytes([block[2], block[3]]));
1033 let out_base = b * BLOCK_SIZE;
1034 for j in 0..16 {
1035 let byte = block[4 + j];
1036 let lo = (byte & 0x0F) as f32;
1037 let hi = ((byte >> 4) & 0x0F) as f32;
1038 result[out_base + j * 2] = d * lo + m;
1039 result[out_base + j * 2 + 1] = d * hi + m;
1040 }
1041 }
1042
1043 result.truncate(num_elements);
1044 result
1045}
1046
1047#[cfg(test)]
1048mod q5k_dequant_tests {
1049 use super::dequantize_q5k_to_f32;
1050
1051 fn build_q5k_block(
1057 d: u16,
1058 dmin: u16,
1059 scale_bytes: [u8; 12],
1060 qh: [u8; 32],
1061 qs: [u8; 128],
1062 ) -> Vec<u8> {
1063 let mut block = Vec::with_capacity(176);
1064 block.extend_from_slice(&d.to_le_bytes());
1065 block.extend_from_slice(&dmin.to_le_bytes());
1066 block.extend_from_slice(&scale_bytes);
1067 block.extend_from_slice(&qh);
1068 block.extend_from_slice(&qs);
1069 assert_eq!(block.len(), 176, "Q5_K super-block must be 176 bytes");
1070 block
1071 }
1072
1073 fn scales_all_one_mins_zero() -> [u8; 12] {
1076 [1, 1, 1, 1, 0, 0, 0, 0, 1, 1, 1, 1]
1083 }
1084
1085 #[test]
1091 fn test_q5k_dequant_stride32_nibble_layout() {
1092 let d = 0x3C00; let dmin = 0x0000; let mut qs = [0u8; 128];
1095 qs[0] = 0x21; let block = build_q5k_block(d, dmin, scales_all_one_mins_zero(), [0u8; 32], qs);
1097
1098 let out = dequantize_q5k_to_f32(&block, 256);
1099
1100 assert_eq!(out[0], 1.0, "out[0] must come from qs[0] low nibble (=1)");
1102 assert_eq!(out[1], 0.0, "out[1] must come from qs[1] low nibble (=0)");
1104 assert_eq!(out[32], 2.0, "out[32] must come from qs[0] high nibble (=2)");
1106 }
1107
1108 #[test]
1114 fn test_q5k_dequant_fifth_bit_placement() {
1115 let d = 0x3C00; let dmin = 0x0000; let mut qs = [0u8; 128];
1118 qs[0] = 0x21; let mut qh = [0u8; 32];
1120 qh[0] = 0x03; let block = build_q5k_block(d, dmin, scales_all_one_mins_zero(), qh, qs);
1122
1123 let out = dequantize_q5k_to_f32(&block, 256);
1124
1125 assert_eq!(out[0], 17.0, "out[0] low nibble must add 16 for qh bit 0");
1127 assert_eq!(out[32], 18.0, "out[32] high nibble must add 16 for qh bit 1");
1129 }
1130}