1use crate::error::EngineError;
9use crate::model::ModelEngine;
10use spec_stream_core::{Logits, TokenId};
11
12pub const QK_K: usize = 32;
14
15#[inline]
17pub fn f16_to_f32(h: u16) -> f32 {
18 let sign = ((h >> 15) & 1) as u32;
19 let exp = ((h >> 10) & 0x1F) as u32;
20 let mant = (h & 0x3FF) as u32;
21
22 let f_sign = sign << 31;
23 if exp == 0 {
24 if mant == 0 {
25 return f32::from_bits(f_sign);
26 }
27 let mut m = mant;
29 let mut e = 0;
30 while (m & 0x400) == 0 {
31 m <<= 1;
32 e += 1;
33 }
34 m &= 0x3FF;
35 let f_exp = (127 - 15 + 1 - e) << 23;
36 let f_mant = m << 13;
37 return f32::from_bits(f_sign | f_exp | f_mant);
38 } else if exp == 0x1F {
39 let f_exp = 0xFF << 23;
40 let f_mant = mant << 13;
41 return f32::from_bits(f_sign | f_exp | f_mant);
42 }
43
44 let f_exp = (exp + 127 - 15) << 23;
45 let f_mant = mant << 13;
46 f32::from_bits(f_sign | f_exp | f_mant)
47}
48
49#[inline]
51pub fn f32_to_f16(val: f32) -> u16 {
52 let bits = val.to_bits();
53 let sign = (bits >> 31) & 1;
54 let exp = ((bits >> 23) & 0xFF) as i32;
55 let mant = bits & 0x7FFFFF;
56
57 if exp == 0xFF {
58 let h_mant = if mant != 0 { 0x200 } else { 0 };
60 return ((sign << 15) | (0x1F << 10) | h_mant) as u16;
61 }
62
63 let new_exp = exp - 127 + 15;
64 if new_exp >= 31 {
65 return ((sign << 15) | (0x1F << 10)) as u16;
66 } else if new_exp <= 0 {
67 if new_exp < -10 {
68 return (sign << 15) as u16;
69 }
70 let m = (mant | 0x800000) >> (1 - new_exp);
71 return ((sign << 15) | (m >> 13)) as u16;
72 }
73
74 ((sign << 15) | ((new_exp as u32) << 10) | (mant >> 13)) as u16
75}
76
77#[derive(Debug, Clone, Copy, PartialEq, Eq)]
79pub enum QuantType {
80 Q8_0,
82 Q4_0,
84}
85
86pub fn dequantize_q8_0(src: &[u8], dst: &mut [f32]) {
88 assert_eq!(dst.len() % QK_K, 0, "Destination must be multiple of 32");
89 let num_blocks = dst.len() / QK_K;
90 assert_eq!(
91 src.len(),
92 num_blocks * 34,
93 "Source length mismatch for Q8_0"
94 );
95
96 for b in 0..num_blocks {
97 let src_offset = b * 34;
98 let scale_bytes = [src[src_offset], src[src_offset + 1]];
99 let scale = f16_to_f32(u16::from_le_bytes(scale_bytes));
100
101 let dst_offset = b * QK_K;
102 for i in 0..QK_K {
103 let q = src[src_offset + 2 + i] as i8;
104 dst[dst_offset + i] = q as f32 * scale;
105 }
106 }
107}
108
109pub fn dequantize_q4_0(src: &[u8], dst: &mut [f32]) {
111 assert_eq!(dst.len() % QK_K, 0, "Destination must be multiple of 32");
112 let num_blocks = dst.len() / QK_K;
113 assert_eq!(
114 src.len(),
115 num_blocks * 18,
116 "Source length mismatch for Q4_0"
117 );
118
119 for b in 0..num_blocks {
120 let src_offset = b * 18;
121 let scale_bytes = [src[src_offset], src[src_offset + 1]];
122 let scale = f16_to_f32(u16::from_le_bytes(scale_bytes));
123
124 let dst_offset = b * QK_K;
125 for i in 0..16 {
126 let byte = src[src_offset + 2 + i];
127 let q0 = (byte & 0x0F) as i8 - 8;
128 let q1 = ((byte >> 4) & 0x0F) as i8 - 8;
129
130 dst[dst_offset + i] = q0 as f32 * scale;
131 dst[dst_offset + i + 16] = q1 as f32 * scale;
132 }
133 }
134}
135
136pub fn vec_dot_q8_0(w_bytes: &[u8], x: &[f32]) -> f32 {
138 #[cfg(target_arch = "x86_64")]
139 {
140 if is_x86_feature_detected!("avx2") && is_x86_feature_detected!("fma") {
141 return unsafe { vec_dot_q8_0_avx2(w_bytes, x) };
143 }
144 }
145
146 vec_dot_q8_0_scalar(w_bytes, x)
147}
148
149pub fn vec_dot_q8_0_scalar(w_bytes: &[u8], x: &[f32]) -> f32 {
151 let num_blocks = x.len() / QK_K;
152 let mut sum = 0.0f32;
153
154 for b in 0..num_blocks {
155 let src_offset = b * 34;
156 let scale = f16_to_f32(u16::from_le_bytes([
157 w_bytes[src_offset],
158 w_bytes[src_offset + 1],
159 ]));
160 let x_offset = b * QK_K;
161
162 let mut block_acc = 0.0f32;
163 for i in 0..QK_K {
164 let q = w_bytes[src_offset + 2 + i] as i8;
165 block_acc += q as f32 * x[x_offset + i];
166 }
167 sum += block_acc * scale;
168 }
169
170 sum
171}
172
173#[cfg(target_arch = "x86_64")]
174#[target_feature(enable = "avx2,fma")]
175unsafe fn vec_dot_q8_0_avx2(w_bytes: &[u8], x: &[f32]) -> f32 {
176 use std::arch::x86_64::*;
177
178 let num_blocks = x.len() / QK_K;
179 let mut sum = 0.0f32;
180
181 for b in 0..num_blocks {
182 let src_offset = b * 34;
183 let scale = f16_to_f32(u16::from_le_bytes([
184 w_bytes[src_offset],
185 w_bytes[src_offset + 1],
186 ]));
187 let x_ptr = x.as_ptr().add(b * QK_K);
188 let q_ptr = w_bytes.as_ptr().add(src_offset + 2);
189
190 let mut acc = _mm256_setzero_ps();
191
192 for i in 0..4 {
193 let i8_slice = std::slice::from_raw_parts(q_ptr.add(i * 8), 8);
194 let mut i64_buf = [0u8; 8];
195 i64_buf.copy_from_slice(i8_slice);
196 let q_i64 = i64::from_ne_bytes(i64_buf);
197 let q128 = _mm_cvtsi64_si128(q_i64);
198
199 let q256_i32 = _mm256_cvtepi8_epi32(q128);
200 let q256_f32 = _mm256_cvtepi32_ps(q256_i32);
201 let x256_f32 = _mm256_loadu_ps(x_ptr.add(i * 8));
202
203 acc = _mm256_fmadd_ps(q256_f32, x256_f32, acc);
204 }
205
206 let h128 = _mm_add_ps(_mm256_castps256_ps128(acc), _mm256_extractf128_ps(acc, 1));
207 let shuf = _mm_movehdup_ps(h128);
208 let sums = _mm_add_ps(h128, shuf);
209 let shuf2 = _mm_movehl_ps(sums, sums);
210 let total = _mm_add_ss(sums, shuf2);
211 let block_sum = _mm_cvtss_f32(total);
212
213 sum += block_sum * scale;
214 }
215
216 sum
217}
218
219pub fn vec_dot_q4_0(w_bytes: &[u8], x: &[f32]) -> f32 {
221 let num_blocks = x.len() / QK_K;
222 let mut sum = 0.0f32;
223
224 for b in 0..num_blocks {
225 let src_offset = b * 18;
226 let scale = f16_to_f32(u16::from_le_bytes([
227 w_bytes[src_offset],
228 w_bytes[src_offset + 1],
229 ]));
230 let x_offset = b * QK_K;
231
232 let mut block_acc = 0.0f32;
233 for i in 0..16 {
234 let byte = w_bytes[src_offset + 2 + i];
235 let q0 = (byte & 0x0F) as i8 - 8;
236 let q1 = ((byte >> 4) & 0x0F) as i8 - 8;
237
238 block_acc += q0 as f32 * x[x_offset + i];
239 block_acc += q1 as f32 * x[x_offset + i + 16];
240 }
241 sum += block_acc * scale;
242 }
243
244 sum
245}
246
247pub fn quantize_q8_0_block(src: &[f32], dst: &mut [u8]) {
249 assert_eq!(src.len(), QK_K);
250 assert_eq!(dst.len(), 34);
251
252 let mut amax = 0.0f32;
253 for &val in src {
254 amax = amax.max(val.abs());
255 }
256
257 let d = amax / 127.0;
258 let id = if d != 0.0 { 1.0 / d } else { 0.0 };
259
260 let f16_scale = f32_to_f16(d);
261 let scale_bytes = f16_scale.to_le_bytes();
262 dst[0] = scale_bytes[0];
263 dst[1] = scale_bytes[1];
264
265 for i in 0..QK_K {
266 let q = (src[i] * id).round().clamp(-128.0, 127.0) as i8;
267 dst[2 + i] = q as u8;
268 }
269}
270
271#[derive(Debug, Clone)]
274pub struct QuantizedLinearModel {
275 pub name: String,
276 pub vocab_size: usize,
277 pub hidden_dim: usize,
278 pub quant_type: QuantType,
279 pub quantized_weights: Vec<u8>,
281 pub bytes_per_row: usize,
282 pub embeddings: Vec<f32>,
284}
285
286impl QuantizedLinearModel {
287 pub fn from_f32_weights(
289 name: impl Into<String>,
290 vocab_size: usize,
291 hidden_dim: usize,
292 weights: &[f32],
293 embeddings: Vec<f32>,
294 ) -> Result<Self, EngineError> {
295 if hidden_dim % QK_K != 0 {
296 return Err(EngineError::InvalidConfig(format!(
297 "Hidden dimension {} must be a multiple of {}",
298 hidden_dim, QK_K
299 )));
300 }
301
302 let num_blocks = hidden_dim / QK_K;
303 let bytes_per_row = num_blocks * 34; let mut quantized_weights = vec![0u8; vocab_size * bytes_per_row];
305
306 for row in 0..vocab_size {
307 let row_offset_f32 = row * hidden_dim;
308 let row_offset_bytes = row * bytes_per_row;
309
310 for b in 0..num_blocks {
311 let src_slice =
312 &weights[row_offset_f32 + b * QK_K..row_offset_f32 + (b + 1) * QK_K];
313 let dst_slice = &mut quantized_weights
314 [row_offset_bytes + b * 34..row_offset_bytes + (b + 1) * 34];
315 quantize_q8_0_block(src_slice, dst_slice);
316 }
317 }
318
319 Ok(Self {
320 name: name.into(),
321 vocab_size,
322 hidden_dim,
323 quant_type: QuantType::Q8_0,
324 quantized_weights,
325 bytes_per_row,
326 embeddings,
327 })
328 }
329
330 pub fn project_hidden(&self, hidden: &[f32]) -> Logits {
332 assert_eq!(hidden.len(), self.hidden_dim);
333 let mut logits = Vec::with_capacity(self.vocab_size);
334
335 for row in 0..self.vocab_size {
336 let offset = row * self.bytes_per_row;
337 let row_bytes = &self.quantized_weights[offset..offset + self.bytes_per_row];
338
339 let dot = match self.quant_type {
340 QuantType::Q8_0 => vec_dot_q8_0(row_bytes, hidden),
341 QuantType::Q4_0 => vec_dot_q4_0(row_bytes, hidden),
342 };
343 logits.push(dot);
344 }
345
346 Logits::new(logits)
347 }
348
349 pub fn project_token(&self, token: TokenId) -> Logits {
351 let token_idx = (token as usize) % self.vocab_size;
352 let emb_offset = token_idx * self.hidden_dim;
353 let hidden = &self.embeddings[emb_offset..emb_offset + self.hidden_dim];
354 self.project_hidden(hidden)
355 }
356}
357
358impl ModelEngine for QuantizedLinearModel {
359 async fn forward_batch(&mut self, tokens: &[TokenId]) -> Result<Vec<Logits>, EngineError> {
360 let mut out = Vec::with_capacity(tokens.len());
361 for &tok in tokens {
362 out.push(self.project_token(tok));
363 }
364 Ok(out)
365 }
366
367 async fn forward_token(&mut self, token: TokenId) -> Result<Logits, EngineError> {
368 Ok(self.project_token(token))
369 }
370}
371
372#[cfg(test)]
373mod tests {
374 use super::*;
375
376 #[test]
377 fn test_f16_to_f32_roundtrip() {
378 let test_vals = [0.0f32, 1.0, -1.0, 0.5, 2.5, -15.25, 100.0];
379 for &v in &test_vals {
380 let h = f32_to_f16(v);
381 let back = f16_to_f32(h);
382 assert!((v - back).abs() < 1e-2, "Failed for {v}: got {back}");
383 }
384 }
385
386 #[test]
387 fn test_q8_0_quantize_dequantize() {
388 let mut original = [0.0f32; 32];
389 for (i, val) in original.iter_mut().enumerate() {
390 *val = (i as f32 - 16.0) * 0.25;
391 }
392
393 let mut block = [0u8; 34];
394 quantize_q8_0_block(&original, &mut block);
395
396 let mut dequantized = [0.0f32; 32];
397 dequantize_q8_0(&block, &mut dequantized);
398
399 for i in 0..32 {
400 assert!(
401 (original[i] - dequantized[i]).abs() < 0.05,
402 "Diff too large at {i}: orig {}, got {}",
403 original[i],
404 dequantized[i]
405 );
406 }
407 }
408
409 #[test]
410 fn test_quantized_linear_model() {
411 let vocab_size = 16;
412 let hidden_dim = 32;
413 let size = vocab_size * hidden_dim;
414
415 let weights: Vec<f32> = (0..size).map(|i| (i as f32 % 10.0) * 0.1).collect();
416 let embeddings: Vec<f32> = (0..size).map(|i| ((i + 5) as f32 % 10.0) * 0.1).collect();
417
418 let model = QuantizedLinearModel::from_f32_weights(
419 "test-q8", vocab_size, hidden_dim, &weights, embeddings,
420 )
421 .expect("Failed to build quantized model");
422
423 assert_eq!(model.bytes_per_row, 34);
424 assert_eq!(model.quantized_weights.len(), 16 * 34);
425
426 let logits = model.project_token(2);
427 assert_eq!(logits.vocab_size(), 16);
428 }
429
430 #[test]
431 fn test_vec_dot_q8_0_simd_equivalence() {
432 let mut original = [0.0f32; 64];
433 let mut x = [0.0f32; 64];
434 for i in 0..64 {
435 original[i] = (i as f32 - 32.0) * 0.15;
436 x[i] = (i as f32 % 7.0) * 0.3;
437 }
438
439 let mut w_bytes = vec![0u8; 68]; quantize_q8_0_block(&original[0..32], &mut w_bytes[0..34]);
441 quantize_q8_0_block(&original[32..64], &mut w_bytes[34..68]);
442
443 let scalar_res = vec_dot_q8_0_scalar(&w_bytes, &x);
444 let simd_res = vec_dot_q8_0(&w_bytes, &x);
445
446 assert!(
447 (scalar_res - simd_res).abs() < 1e-4,
448 "SIMD dot product ({simd_res}) must match scalar dot product ({scalar_res})"
449 );
450 }
451}