cortiq_engine/
qtensor.rs

1//! QTensor — weight tensor with pluggable storage.
2//!
3//! Two backings, one interface:
4//! - `F32`   — owned dense floats (small models, tests). Every operation
5//!   is bit-identical to the historical `&[f32]` code paths.
6//! - `Mapped` — quantized bytes zero-copy from the CMF mmap (`q8_row` /
7//!   `q8_2f`). The matvec is fused: int8 rows × f32 activations, the
8//!   q8_2f column field folds into a pre-scale of the input
9//!   (`x'[i] = col[i]·x[i]`), so the inner loop is the same i8 dot as
10//!   q8_row. This is what lets a 15B file run in a few GB of RSS.
11//!
12//! Extension point: new dtypes = new match arm here, nothing else moves.
13
14use crate::pool::{Pool, matvec_rows, matvec_rows2};
15use cortiq_core::quant::{GROUP_SIZE, Q1_TILE, Q4_TILE, f16_to_f32};
16use cortiq_core::{CmfModel, TensorDtype};
17use std::sync::Arc;
18
19pub enum QTensor {
20    F32 {
21        data: Vec<f32>,
22        rows: usize,
23        cols: usize,
24    },
25    Mapped {
26        model: Arc<CmfModel>,
27        /// Index into the model's tensor directory.
28        idx: usize,
29        dtype: TensorDtype,
30        rows: usize,
31        cols: usize,
32        /// Per-row scales, dequantized to f32 up front (tiny).
33        row_scale: Vec<f32>,
34        /// q8_2f column field (θ), dequantized up front; empty for q8_row.
35        col_field: Vec<f32>,
36        /// Vbit only: byte offset of each row's packed data within the
37        /// tensor blob (`[rows + 1]`, computed once at load — the per-
38        /// matvec prefix scan over row bit-widths was O(rows) each call).
39        vbit_offsets: Vec<usize>,
40        /// q8-family decode repack (load-time, optional): rows in groups
41        /// of 4, interleaved in 16-byte units — one 64-byte line per
42        /// iteration feeds all 4 sdot lanes, ONE sequential weight
43        /// stream per worker instead of four (this is where llama.cpp's
44        /// repacked Q8 kernels get their bandwidth). Empty = off
45        /// (CMF_REPACK=0, non-SDOT arch, or an ineligible shape). Trades
46        /// an anonymous copy of the quants for mmap pages that go cold.
47        repack: Vec<u8>,
48    },
49}
50
51/// Load-time q8 repack gate (see `Mapped::repack`). OPT-IN
52/// (`CMF_REPACK=1`): the single-stream hypothesis LOST on Apple Silicon
53/// (M4, interleaved A/B: decode 101 vs 94 tok/s — four adjacent row
54/// streams per worker feed the prefetcher MORE memory-level parallelism
55/// than one); kept as an experiment flag for x86, where the tradeoff
56/// may land differently.
57fn repack_enabled() -> bool {
58    static ON: std::sync::OnceLock<bool> = std::sync::OnceLock::new();
59    *ON.get_or_init(|| {
60        std::env::var("CMF_REPACK")
61            .map(|v| v == "1")
62            .unwrap_or(cfg!(target_os = "android"))
63    })
64}
65
66/// Interleave q8 rows for the decode kernel: group g holds rows
67/// 4g..4g+4 as [r0[c], r1[c], r2[c], r3[c]] per 16-byte chunk c. Only
68/// full groups are packed — tail rows keep reading the mmap layout.
69fn q8_repack(bytes: &[u8], rows: usize, cols: usize) -> Vec<u8> {
70    #[cfg(target_arch = "aarch64")]
71    let arch_ok = sdot_enabled();
72    #[cfg(not(target_arch = "aarch64"))]
73    let arch_ok = false;
74    if !arch_ok || !repack_enabled() || rows < 256 || cols % 16 != 0 {
75        return Vec::new();
76    }
77    q8_repack_layout(bytes, rows, cols)
78}
79
80/// The pure layout transform behind `q8_repack` (tested directly —
81/// the gate depends on arch and env).
82fn q8_repack_layout(bytes: &[u8], rows: usize, cols: usize) -> Vec<u8> {
83    let groups = rows / 4;
84    let mut rep = vec![0u8; groups * 4 * cols];
85    for g in 0..groups {
86        let dst = &mut rep[g * 4 * cols..(g + 1) * 4 * cols];
87        for c in 0..cols / 16 {
88            for lane in 0..4 {
89                let src = (g * 4 + lane) * cols + c * 16;
90                dst[c * 64 + lane * 16..c * 64 + lane * 16 + 16]
91                    .copy_from_slice(&bytes[src..src + 16]);
92            }
93        }
94    }
95    rep
96}
97
98/// Prefix-sum of vbit row payload offsets (absolute within the tensor
99/// bytes). `offsets[r]..offsets[r+1]` is row r's packed data.
100fn vbit_row_offsets(bytes: &[u8], rows: usize, cols: usize) -> Vec<usize> {
101    let ng = cols / GROUP_SIZE;
102    let bits = &bytes[..rows];
103    let mut offsets = Vec::with_capacity(rows + 1);
104    let mut off = rows + rows * ng * 2;
105    for r in 0..rows {
106        offsets.push(off);
107        off += (cols * bits[r] as usize).div_ceil(8);
108    }
109    offsets.push(off);
110    offsets
111}
112
113impl QTensor {
114    pub fn from_f32(data: Vec<f32>, rows: usize, cols: usize) -> Self {
115        debug_assert_eq!(data.len(), rows * cols);
116        Self::F32 { data, rows, cols }
117    }
118
119    /// Wrap a directory tensor without dequantizing the payload.
120    /// Falls back to dequantized f32 for dtypes without a fused kernel.
121    pub fn from_model(model: &Arc<CmfModel>, name: &str) -> Result<Self, String> {
122        // Indexed lookup: the linear directory scan made pipeline build
123        // O(N²) on MoE/skills files with thousands of tensors.
124        let idx = model
125            .tensor_index(name)
126            .ok_or_else(|| format!("tensor '{name}' not found in CMF directory"))?;
127        let entry = &model.tensors[idx];
128        if entry.shape.len() != 2 {
129            return Err(format!("QTensor::from_model needs 2-D, got '{name}'"));
130        }
131        let (rows, cols) = (entry.shape[0], entry.shape[1]);
132        let bytes = model.entry_bytes(entry);
133
134        match entry.dtype {
135            TensorDtype::Q8Row | TensorDtype::Q8_2f => {
136                let n = rows * cols;
137                let scales_off = n;
138                let row_scale: Vec<f32> = (0..rows)
139                    .map(|o| {
140                        f16_to_f32(u16::from_le_bytes([
141                            bytes[scales_off + o * 2],
142                            bytes[scales_off + o * 2 + 1],
143                        ]))
144                    })
145                    .collect();
146                let col_field: Vec<f32> = if entry.dtype == TensorDtype::Q8_2f {
147                    let col_off = n + rows * 2;
148                    (0..cols)
149                        .map(|i| {
150                            f16_to_f32(u16::from_le_bytes([
151                                bytes[col_off + i * 2],
152                                bytes[col_off + i * 2 + 1],
153                            ]))
154                        })
155                        .collect()
156                } else {
157                    Vec::new()
158                };
159                Ok(Self::Mapped {
160                    model: model.clone(),
161                    idx,
162                    dtype: entry.dtype,
163                    rows,
164                    cols,
165                    row_scale,
166                    col_field,
167                    vbit_offsets: Vec::new(),
168                    repack: q8_repack(bytes, rows, cols),
169                })
170            }
171            // vbit: fused kernel unpacks variable-bit rows from mmap.
172            TensorDtype::Vbit if cols % GROUP_SIZE == 0 => Ok(Self::Mapped {
173                model: model.clone(),
174                idx,
175                dtype: entry.dtype,
176                rows,
177                cols,
178                row_scale: Vec::new(),
179                col_field: Vec::new(),
180                vbit_offsets: vbit_row_offsets(bytes, rows, cols),
181                repack: Vec::new(),
182            }),
183            // vbit_ro (§4.2): the offset table comes straight from the
184            // file — no load-time prefix scan; kernels are shared with
185            // legacy vbit (they consume absolute offsets either way).
186            TensorDtype::VbitRo if cols % GROUP_SIZE == 0 => {
187                let (_, off_off, packed_off) = cortiq_core::quant::vbit_ro_sections(rows, cols);
188                let offsets: Vec<usize> = (0..=rows)
189                    .map(|r| packed_off + cortiq_core::quant::vbit_ro_offset(bytes, off_off, r))
190                    .collect();
191                Ok(Self::Mapped {
192                    model: model.clone(),
193                    idx,
194                    dtype: entry.dtype,
195                    rows,
196                    cols,
197                    row_scale: Vec::new(),
198                    col_field: Vec::new(),
199                    vbit_offsets: offsets,
200                    repack: Vec::new(),
201                })
202            }
203            // q4_block: fused kernel reads nibbles straight from mmap —
204            // a 14B q4 file no longer explodes into ×8 f32 RAM.
205            // q4_tiled (§4.3): interleaved [scale][nibbles] tiles — one
206            // sequential memory stream (measured ×1.66 ARM / ×1.13 AVX2
207            // at kernel level over the split layout).
208            TensorDtype::Q4Tiled if cols % GROUP_SIZE == 0 => Ok(Self::Mapped {
209                model: model.clone(),
210                idx,
211                dtype: entry.dtype,
212                rows,
213                cols,
214                row_scale: Vec::new(),
215                col_field: Vec::new(),
216                vbit_offsets: Vec::new(),
217                repack: Vec::new(),
218            }),
219            TensorDtype::Q4Block if cols % GROUP_SIZE == 0 => Ok(Self::Mapped {
220                model: model.clone(),
221                idx,
222                dtype: entry.dtype,
223                rows,
224                cols,
225                row_scale: Vec::new(),
226                col_field: Vec::new(),
227                vbit_offsets: Vec::new(),
228                repack: Vec::new(),
229            }),
230            // q1: binary sign-bit tiles from mmap (1-bit-trained models).
231            TensorDtype::Q1 if cols % GROUP_SIZE == 0 => Ok(Self::Mapped {
232                model: model.clone(),
233                idx,
234                dtype: entry.dtype,
235                rows,
236                cols,
237                row_scale: Vec::new(),
238                col_field: Vec::new(),
239                vbit_offsets: Vec::new(),
240                repack: Vec::new(),
241            }),
242            // q1t (ternary + outlier overlay): fused per-row dequant kernel
243            // reads straight from mmap — a 12B q1t stays ~its file size in
244            // RAM instead of dequantizing to ~48 GB of f32.
245            TensorDtype::Q1T if cols % GROUP_SIZE == 0 => Ok(Self::Mapped {
246                model: model.clone(),
247                idx,
248                dtype: entry.dtype,
249                rows,
250                cols,
251                row_scale: Vec::new(),
252                col_field: Vec::new(),
253                vbit_offsets: Vec::new(),
254                repack: Vec::new(),
255            }),
256            // No fused kernel yet → dequantize once (correct, more RAM).
257            _ => {
258                let mut data = vec![0.0f32; rows * cols];
259                cortiq_core::quant::dequant_tensor(entry, bytes, &mut data)?;
260                Ok(Self::from_f32(data, rows, cols))
261            }
262        }
263    }
264
265    /// q1-mapped tensor? (GPU gates: the q1 CPU kernel is
266    /// compute-bound, so offload pays at much smaller shapes than q8.)
267    pub(crate) fn is_q1(&self) -> bool {
268        matches!(
269            self,
270            Self::Mapped {
271                dtype: TensorDtype::Q1,
272                ..
273            }
274        )
275    }
276
277    /// Owned-f32 view (data, rows, cols) — the GDN a/b gate projections
278    /// arrive dequantized (force-f16 in the converter → F32 in RAM).
279    pub(crate) fn f32_parts(&self) -> Option<(&[f32], usize, usize)> {
280        match self {
281            Self::F32 { data, rows, cols } => Some((data, *rows, *cols)),
282            _ => None,
283        }
284    }
285
286    /// (directory idx, rows, cols) of a q1-mapped tensor — the
287    /// whole-block GPU path resolves offsets itself.
288    /// (idx, rows, cols) of a mapped tensor the whole-token GPU graph can drive
289    /// — Q1, Q1T or Q4-block (it resolves the offset and picks the kernel by
290    /// dtype). Q4-block lets a precise down_proj/lm_head stay on-device.
291    /// Named `q1_parts` for historical reasons.
292    pub(crate) fn q1_parts(&self) -> Option<(usize, usize, usize)> {
293        match self {
294            #[cfg(target_os = "macos")]
295            Self::Mapped {
296                dtype: TensorDtype::Q1T,
297                ..
298            } if !crate::gpu::metal_q1t_enabled() => None,
299            Self::Mapped {
300                idx,
301                dtype:
302                    TensorDtype::Q1
303                    | TensorDtype::Q1T
304                    | TensorDtype::Q4Block
305                    | TensorDtype::Q4Tiled
306                    | TensorDtype::Q8Row
307                    | TensorDtype::Q8_2f,
308                rows,
309                cols,
310                ..
311            } => Some((*idx, *rows, *cols)),
312            _ => None,
313        }
314    }
315
316    /// (directory idx, rows, cols, row_scale) of a plain q8_row mapped
317    /// tensor — the chunk-prefill GPU graph resolves offsets itself.
318    /// q8_2f is excluded on purpose: its column field would need a
319    /// prescale stage on the device.
320    pub(crate) fn q8_row_parts(&self) -> Option<(usize, usize, usize, &[f32])> {
321        match self {
322            Self::Mapped {
323                idx,
324                dtype: TensorDtype::Q8Row,
325                rows,
326                cols,
327                row_scale,
328                col_field,
329                ..
330            } if col_field.is_empty() => Some((*idx, *rows, *cols, row_scale)),
331            _ => None,
332        }
333    }
334
335    pub fn rows(&self) -> usize {
336        match self {
337            Self::F32 { rows, .. } | Self::Mapped { rows, .. } => *rows,
338        }
339    }
340
341    /// Mapped q4t handle (model + directory index) — the fused GPU FFN
342    /// needs the raw file coordinates of its three projections.
343    pub(crate) fn mapped_q4t(&self) -> Option<(&Arc<CmfModel>, usize)> {
344        match self {
345            Self::Mapped {
346                model,
347                idx,
348                dtype: TensorDtype::Q4Tiled,
349                ..
350            } => Some((model, *idx)),
351            _ => None,
352        }
353    }
354
355    pub fn cols(&self) -> usize {
356        match self {
357            Self::F32 { cols, .. } | Self::Mapped { cols, .. } => *cols,
358        }
359    }
360
361    /// (model, tensor idx) for a q1 mapped weight — the wgpu token graph
362    /// keys its resident VRAM cache by idx. None for any other dtype/kind.
363    pub fn mapped_q1(&self) -> Option<(&std::sync::Arc<CmfModel>, usize)> {
364        match self {
365            Self::Mapped {
366                model,
367                idx,
368                dtype: TensorDtype::Q1,
369                ..
370            } => Some((model, *idx)),
371            _ => None,
372        }
373    }
374
375    /// (model, idx, kind, row_scale) for a graph-capable mapped weight. kind:
376    /// 0=q8_row (per-row scales), 1=q1, 2=q4_tiled, 3=q1t (tile-embedded, no
377    /// rs). None for dtypes the token graph does not handle (q8_2f/q4_block/vbit).
378    pub fn graph_weight(&self) -> Option<(&std::sync::Arc<CmfModel>, usize, u8, &[f32])> {
379        match self {
380            Self::Mapped {
381                model,
382                idx,
383                dtype: TensorDtype::Q8Row,
384                row_scale,
385                ..
386            } => Some((model, *idx, 0, row_scale.as_slice())),
387            Self::Mapped {
388                model,
389                idx,
390                dtype: TensorDtype::Q1,
391                ..
392            } => Some((model, *idx, 1, &[])),
393            // Q4Tiled is kind 5, NOT 2: both carried 2 historically, and
394            // the wgpu token graph fed 18B interleaved tiles to the
395            // split-layout q4b kernel — garbage output on q4t models
396            // (caught by an end-to-end answer check on real Vulkan).
397            Self::Mapped {
398                model,
399                idx,
400                dtype: TensorDtype::Q4Tiled,
401                ..
402            } => Some((model, *idx, 5, &[])),
403            Self::Mapped {
404                model,
405                idx,
406                dtype: TensorDtype::Q4Block,
407                ..
408            } => Some((model, *idx, 2, &[])),
409            Self::Mapped {
410                model,
411                idx,
412                dtype: TensorDtype::Q1T,
413                ..
414            } => Some((model, *idx, 3, &[])),
415            _ => None,
416        }
417    }
418
419    /// Dense f32 view — only for owned tensors. Masked/sparse execution
420    /// paths require it; quantized weights don't support masks yet.
421    pub fn as_f32(&self) -> Option<&[f32]> {
422        match self {
423            Self::F32 { data, .. } => Some(data),
424            Self::Mapped { .. } => None,
425        }
426    }
427
428    fn quant_bytes(&self) -> &[u8] {
429        match self {
430            Self::Mapped { model, idx, .. } => model.entry_bytes(&model.tensors[*idx]),
431            Self::F32 { .. } => unreachable!("quant_bytes on F32"),
432        }
433    }
434
435    /// Dequantize one row into `dst` (embedding lookup).
436    pub fn row_f32(&self, r: usize, dst: &mut [f32]) {
437        let cols = self.cols();
438        debug_assert_eq!(dst.len(), cols);
439        match self {
440            Self::F32 { data, .. } => dst.copy_from_slice(&data[r * cols..(r + 1) * cols]),
441            Self::Mapped {
442                dtype,
443                row_scale,
444                col_field,
445                vbit_offsets,
446                ..
447            } => {
448                if *dtype == TensorDtype::Q4Tiled {
449                    let bytes = self.quant_bytes();
450                    let gpr = cols / GROUP_SIZE;
451                    for gi in 0..gpr {
452                        let tile = &bytes[(r * gpr + gi) * Q4_TILE..(r * gpr + gi + 1) * Q4_TILE];
453                        let s = f16_to_f32(u16::from_le_bytes([tile[0], tile[1]]));
454                        for (k, &b) in tile[2..].iter().enumerate() {
455                            dst[gi * GROUP_SIZE + k * 2] = ((b & 0x0F) as f32 - 8.0) * s;
456                            dst[gi * GROUP_SIZE + k * 2 + 1] = (((b >> 4) & 0x0F) as f32 - 8.0) * s;
457                        }
458                    }
459                    return;
460                }
461                if *dtype == TensorDtype::Q4Block {
462                    let (packed, scales) = q4_split(self.quant_bytes(), self.rows(), cols);
463                    let gpr = cols / GROUP_SIZE;
464                    for gi in 0..gpr {
465                        let g = r * gpr + gi;
466                        let s = f16_to_f32(u16::from_le_bytes([scales[g * 2], scales[g * 2 + 1]]));
467                        for (k, &b) in packed[g * 16..(g + 1) * 16].iter().enumerate() {
468                            dst[gi * GROUP_SIZE + k * 2] = ((b & 0x0F) as f32 - 8.0) * s;
469                            dst[gi * GROUP_SIZE + k * 2 + 1] = (((b >> 4) & 0x0F) as f32 - 8.0) * s;
470                        }
471                    }
472                    return;
473                }
474                if *dtype == TensorDtype::Q1 {
475                    let bytes = self.quant_bytes();
476                    let gpr = cols / GROUP_SIZE;
477                    for gi in 0..gpr {
478                        let tile = &bytes[(r * gpr + gi) * Q1_TILE..(r * gpr + gi + 1) * Q1_TILE];
479                        let s = f16_to_f32(u16::from_le_bytes([tile[0], tile[1]]));
480                        for (j, &b) in tile[2..].iter().enumerate() {
481                            for k in 0..8 {
482                                dst[gi * GROUP_SIZE + j * 8 + k] =
483                                    (((b >> k) & 1) as f32 * 2.0 - 1.0) * s;
484                            }
485                        }
486                    }
487                    return;
488                }
489                if *dtype == TensorDtype::Q1T {
490                    let bytes = self.quant_bytes();
491                    let gpr = cols / GROUP_SIZE;
492                    let base_len = self.rows() * gpr * cortiq_core::quant::Q1T_TILE;
493                    for gi in 0..gpr {
494                        let off = (r * gpr + gi) * cortiq_core::quant::Q1T_TILE;
495                        let s = cortiq_core::quant::f16_to_f32(u16::from_le_bytes([
496                            bytes[off],
497                            bytes[off + 1],
498                        ]));
499                        let codes = &bytes[off + 2..off + cortiq_core::quant::Q1T_TILE];
500                        for k in 0..GROUP_SIZE {
501                            dst[gi * GROUP_SIZE + k] = match cortiq_core::quant::q1t_code(codes, k)
502                            {
503                                1 => s,
504                                2 => -s,
505                                _ => 0.0,
506                            };
507                        }
508                    }
509                    // Overlay
510                    let rows = self.rows();
511                    let entries = base_len + (rows + 1) * 4;
512                    if entries <= bytes.len() {
513                        let ptrs = &bytes[base_len..base_len + (rows + 1) * 4];
514                        let r0 = u32::from_le_bytes([
515                            ptrs[r * 4],
516                            ptrs[r * 4 + 1],
517                            ptrs[r * 4 + 2],
518                            ptrs[r * 4 + 3],
519                        ]) as usize;
520                        let r1 = u32::from_le_bytes([
521                            ptrs[(r + 1) * 4],
522                            ptrs[(r + 1) * 4 + 1],
523                            ptrs[(r + 1) * 4 + 2],
524                            ptrs[(r + 1) * 4 + 3],
525                        ]) as usize;
526                        let off = entries + r0 * 4;
527                        for i in 0..r1 - r0 {
528                            let item = &bytes[off + i * 4..off + i * 4 + 4];
529                            let c = u16::from_le_bytes([item[0], item[1]]) as usize;
530                            let v = cortiq_core::quant::f16_to_f32(u16::from_le_bytes([
531                                item[2], item[3],
532                            ]));
533                            if c < cols {
534                                dst[c] = v;
535                            }
536                        }
537                    }
538                    return;
539                }
540                if matches!(dtype, TensorDtype::Vbit | TensorDtype::VbitRo) {
541                    let bytes = self.quant_bytes();
542                    let rows = self.rows();
543                    let ng = cols / GROUP_SIZE;
544                    let bits = &bytes[..rows];
545                    let sc_off = rows;
546                    // Precomputed at load — embedding lookup used to scan
547                    // the bit-widths of every preceding row (O(token_id)).
548                    let off = vbit_offsets[r];
549                    let b = bits[r] as usize;
550                    let l = ((1usize << (b - 1)) - 1) as f32;
551                    let data = &bytes[off..];
552                    let (mut acc, mut nbits, mut idx) = (0u64, 0usize, 0usize);
553                    for (i, d) in dst.iter_mut().enumerate() {
554                        while nbits < b {
555                            acc = (acc << 8) | data[idx] as u64;
556                            idx += 1;
557                            nbits += 8;
558                        }
559                        let u = ((acc >> (nbits - b)) & ((1u64 << b) - 1)) as f32;
560                        nbits -= b;
561                        let so = (r * ng + i / GROUP_SIZE) * 2;
562                        let sv = f16_to_f32(u16::from_le_bytes([
563                            bytes[sc_off + so],
564                            bytes[sc_off + so + 1],
565                        ]));
566                        *d = (u - l) * sv;
567                    }
568                    return;
569                }
570                let q = &self.quant_bytes()[r * cols..(r + 1) * cols];
571                let s = row_scale[r];
572                match dtype {
573                    TensorDtype::Q8Row => {
574                        for (d, &b) in dst.iter_mut().zip(q) {
575                            *d = (b as i8) as f32 * s;
576                        }
577                    }
578                    TensorDtype::Q8_2f => {
579                        for (i, (d, &b)) in dst.iter_mut().zip(q).enumerate() {
580                            *d = (b as i8) as f32 * s * col_field[i];
581                        }
582                    }
583                    _ => unreachable!(),
584                }
585            }
586        }
587    }
588
589    /// Can this tensor's columns be read cheaply (for sparse down_proj)?
590    /// True for F32/Q8Row/Q8_2f (per-row scale, direct strided access);
591    /// false for group-packed q4/vbit (column access would unpack whole
592    /// groups — sparse execution falls back to f32 for those).
593    pub fn sparse_col_ok(&self) -> bool {
594        match self {
595            Self::F32 { .. } => true,
596            Self::Mapped { dtype, .. } => {
597                matches!(dtype, TensorDtype::Q8Row | TensorDtype::Q8_2f)
598            }
599        }
600    }
601
602    /// down_proj [hidden, inter]: accumulate `w · col(c)` into `out`
603    /// [hidden] — reads ONLY column `c` (one neuron) from the mmap,
604    /// no full-matrix dequant. `out[k] += w · down[k, c]`.
605    pub fn add_col_scaled(&self, c: usize, w: f32, out: &mut [f32]) {
606        let inter = self.cols();
607        let hidden = self.rows();
608        debug_assert_eq!(out.len(), hidden);
609        match self {
610            Self::F32 { data, .. } => {
611                for (k, o) in out.iter_mut().enumerate() {
612                    *o += w * data[k * inter + c];
613                }
614            }
615            Self::Mapped {
616                dtype,
617                row_scale,
618                col_field,
619                ..
620            } => {
621                let q = self.quant_bytes();
622                let colf = if *dtype == TensorDtype::Q8_2f {
623                    col_field[c]
624                } else {
625                    1.0
626                };
627                let wc = w * colf;
628                for (k, o) in out.iter_mut().enumerate() {
629                    let b = q[k * inter + c] as i8 as f32;
630                    *o += wc * b * row_scale[k];
631                }
632            }
633        }
634    }
635
636    /// Dot of row `r` with `x` (gate/up active-neuron path). Reads only
637    /// row `r` from the mmap — no full dequant. q4/vbit dequant the row
638    /// into `scratch` first (rare for active-FFN weights).
639    pub fn row_dot(&self, r: usize, x: &[f32], scratch: &mut [f32]) -> f32 {
640        let cols = self.cols();
641        match self {
642            Self::F32 { data, .. } => {
643                let row = &data[r * cols..(r + 1) * cols];
644                row.iter().zip(x).map(|(w, v)| w * v).sum()
645            }
646            Self::Mapped {
647                dtype,
648                row_scale,
649                col_field,
650                ..
651            } => match dtype {
652                TensorDtype::Q8Row => {
653                    let q = &self.quant_bytes()[r * cols..(r + 1) * cols];
654                    dot_i8_f32(q, x) * row_scale[r]
655                }
656                TensorDtype::Q8_2f => {
657                    let q = &self.quant_bytes()[r * cols..(r + 1) * cols];
658                    dot_i8_col_f32(q, x, col_field) * row_scale[r]
659                }
660                _ => {
661                    self.row_f32(r, scratch);
662                    scratch.iter().zip(x).map(|(w, v)| w * v).sum()
663                }
664            },
665        }
666    }
667
668    /// `out = W · x` (row-major). F32 delegates to the historical
669    /// bit-exact path; Mapped runs the fused int8 kernel.
670    pub fn matvec(&self, x: &[f32], out: &mut [f32], pool: Option<&Pool>) {
671        match self {
672            Self::F32 { data, .. } => matvec_rows(pool, data, x, out),
673            Self::Mapped {
674                model,
675                idx,
676                dtype,
677                rows,
678                cols,
679                row_scale,
680                col_field,
681                vbit_offsets,
682                repack,
683            } => {
684                let _ = (model, idx);
685                if *dtype == TensorDtype::Q4Block {
686                    // GPU route (wgpu q4b kernel) for large q4_block matvecs —
687                    // gives NVIDIA/AMD/Intel q4 models a GPU path. Probe keeps
688                    // the winner; Metal returns false → the CPU kernel below.
689                    if *rows * *cols >= 8_388_608 && crate::gpu::enabled_here() {
690                        let t0 = std::time::Instant::now();
691                        match crate::gpu::probe_arm(crate::gpu::OpClass::Matvec) {
692                            crate::gpu::ProbeArm::Gpu => {
693                                if crate::gpu::q4b_matvec(model, *idx, x, *rows, *cols, out) {
694                                    crate::gpu::probe_record(
695                                        crate::gpu::OpClass::Matvec,
696                                        true,
697                                        t0.elapsed(),
698                                    );
699                                    return;
700                                }
701                            }
702                            crate::gpu::ProbeArm::CpuTimed => {
703                                q4matvec(self.quant_bytes(), x, *rows, *cols, out, pool);
704                                crate::gpu::probe_record(
705                                    crate::gpu::OpClass::Matvec,
706                                    false,
707                                    t0.elapsed(),
708                                );
709                                return;
710                            }
711                            crate::gpu::ProbeArm::Cpu => {}
712                        }
713                    }
714                    q4matvec(self.quant_bytes(), x, *rows, *cols, out, pool);
715                    return;
716                }
717                if *dtype == TensorDtype::Q4Tiled {
718                    q4t_matvec(self.quant_bytes(), x, *rows, *cols, out, pool);
719                    return;
720                }
721                if *dtype == TensorDtype::Q1 {
722                    // GPU route for large q1 matvecs (out_proj / lm_head
723                    // class): the CPU q1 kernel is load-port-bound at
724                    // ~4 GB/s/core, the GPU one is bandwidth-bound — the
725                    // probe measures both arms and keeps the winner.
726                    if *rows * *cols >= 8_388_608 && crate::gpu::enabled_here() {
727                        let t0 = std::time::Instant::now();
728                        let arm = if crate::gpu::q1_force() {
729                            crate::gpu::ProbeArm::Gpu
730                        } else {
731                            crate::gpu::probe_arm(crate::gpu::OpClass::Matvec)
732                        };
733                        match arm {
734                            crate::gpu::ProbeArm::Gpu => {
735                                if crate::gpu::q1_matvec(model, *idx, x, *rows, *cols, out) {
736                                    crate::gpu::probe_record(
737                                        crate::gpu::OpClass::Matvec,
738                                        true,
739                                        t0.elapsed(),
740                                    );
741                                    return;
742                                }
743                            }
744                            crate::gpu::ProbeArm::CpuTimed => {
745                                q1_matvec(self.quant_bytes(), x, *rows, *cols, out, pool);
746                                crate::gpu::probe_record(
747                                    crate::gpu::OpClass::Matvec,
748                                    false,
749                                    t0.elapsed(),
750                                );
751                                return;
752                            }
753                            crate::gpu::ProbeArm::Cpu => {}
754                        }
755                    }
756                    q1_matvec(self.quant_bytes(), x, *rows, *cols, out, pool);
757                    return;
758                }
759                if *dtype == TensorDtype::Q1T {
760                    // GPU route for large q1t matvecs: the ternary BASE dot runs
761                    // on the GPU (load-port-bound on CPU, like q1), then the
762                    // sparse overlay is added on the CPU. Probe keeps the winner.
763                    if *rows * *cols >= 8_388_608 && crate::gpu::enabled_here() {
764                        let t0 = std::time::Instant::now();
765                        match crate::gpu::probe_arm(crate::gpu::OpClass::Matvec) {
766                            crate::gpu::ProbeArm::Gpu => {
767                                if crate::gpu::q1t_matvec(model, *idx, x, *rows, *cols, out) {
768                                    q1t_add_overlay(self.quant_bytes(), x, *rows, *cols, out, pool);
769                                    crate::gpu::probe_record(
770                                        crate::gpu::OpClass::Matvec,
771                                        true,
772                                        t0.elapsed(),
773                                    );
774                                    return;
775                                }
776                            }
777                            crate::gpu::ProbeArm::CpuTimed => {
778                                q1t_matvec(self.quant_bytes(), x, *rows, *cols, out, pool);
779                                crate::gpu::probe_record(
780                                    crate::gpu::OpClass::Matvec,
781                                    false,
782                                    t0.elapsed(),
783                                );
784                                return;
785                            }
786                            crate::gpu::ProbeArm::Cpu => {}
787                        }
788                    }
789                    q1t_matvec(self.quant_bytes(), x, *rows, *cols, out, pool);
790                    return;
791                }
792                if matches!(dtype, TensorDtype::Vbit | TensorDtype::VbitRo) {
793                    vbitmatvec(self.quant_bytes(), vbit_offsets, x, *rows, *cols, out, pool);
794                    return;
795                }
796                let xs = prescale(x, col_field, *dtype);
797                // D5: large q8 matrices (lm_head-class) — hybrid
798                // CPU∥GPU: split the rows, both sides compute
799                // SIMULTANEOUSLY (same math, shared prescale).
800                // GPU share: CMF_GPU_SPLIT (0..1, default 0.5).
801                if *rows >= crate::gpu::min_rows()
802                    && matches!(dtype, TensorDtype::Q8Row | TensorDtype::Q8_2f)
803                    && std::env::var("CMF_GPU_LMHEAD")
804                        .map(|v| v != "0")
805                        .unwrap_or(true)
806                    && crate::gpu::enabled_here()
807                {
808                    // Runtime probe: alternate the hybrid against the
809                    // pure-CPU matvec, keep whichever is faster HERE.
810                    let t0 = std::time::Instant::now();
811                    match crate::gpu::probe_arm(crate::gpu::OpClass::Matvec) {
812                        crate::gpu::ProbeArm::Gpu => {}
813                        crate::gpu::ProbeArm::CpuTimed => {
814                            qmatvec(
815                                self.quant_bytes(),
816                                repack,
817                                row_scale,
818                                x,
819                                col_field,
820                                *dtype,
821                                *rows,
822                                *cols,
823                                out,
824                                pool,
825                            );
826                            crate::gpu::probe_record(
827                                crate::gpu::OpClass::Matvec,
828                                false,
829                                t0.elapsed(),
830                            );
831                            return;
832                        }
833                        crate::gpu::ProbeArm::Cpu => {
834                            qmatvec(
835                                self.quant_bytes(),
836                                repack,
837                                row_scale,
838                                x,
839                                col_field,
840                                *dtype,
841                                *rows,
842                                *cols,
843                                out,
844                                pool,
845                            );
846                            return;
847                        }
848                    }
849                    let frac = std::env::var("CMF_GPU_SPLIT")
850                        .ok()
851                        .and_then(|v| v.parse::<f32>().ok())
852                        .unwrap_or(0.5)
853                        .clamp(0.0, 1.0);
854                    let cpu_rows = ((*rows as f32) * (1.0 - frac)) as usize;
855                    let (out_cpu, out_gpu) = out.split_at_mut(cpu_rows);
856                    let bytes = self.quant_bytes();
857                    let ok = std::thread::scope(|sc| {
858                        let g = sc.spawn(|| {
859                            crate::gpu::q8_matvec_range(
860                                model,
861                                *idx,
862                                cpu_rows,
863                                &row_scale[cpu_rows..],
864                                &xs,
865                                *rows - cpu_rows,
866                                *cols,
867                                out_gpu,
868                            )
869                        });
870                        if cpu_rows > 0 {
871                            // Repack prefix covers the full groups of the
872                            // CPU half (the split starts at row 0).
873                            let rep_cpu = if repack.is_empty() {
874                                &[][..]
875                            } else {
876                                &repack[..(cpu_rows / 4) * 4 * *cols]
877                            };
878                            qmatvec(
879                                &bytes[..cpu_rows * *cols],
880                                rep_cpu,
881                                &row_scale[..cpu_rows],
882                                x,
883                                col_field,
884                                *dtype,
885                                cpu_rows,
886                                *cols,
887                                out_cpu,
888                                pool,
889                            );
890                        }
891                        g.join().unwrap_or(false)
892                    });
893                    if ok {
894                        crate::gpu::probe_record(crate::gpu::OpClass::Matvec, true, t0.elapsed());
895                        return;
896                    }
897                    // GPU failed — CPU finishes its half (rows rebased —
898                    // group offsets don't line up, mmap layout only).
899                    qmatvec(
900                        &bytes[cpu_rows * *cols..(*rows) * *cols],
901                        &[],
902                        &row_scale[cpu_rows..],
903                        x,
904                        col_field,
905                        *dtype,
906                        *rows - cpu_rows,
907                        *cols,
908                        out_gpu,
909                        pool,
910                    );
911                    return;
912                }
913                qmatvec(
914                    self.quant_bytes(),
915                    repack,
916                    row_scale,
917                    x,
918                    col_field,
919                    *dtype,
920                    *rows,
921                    *cols,
922                    out,
923                    pool,
924                );
925            }
926        }
927    }
928
929    /// Fused two-input matvec (MTP verify pair): weights streamed once.
930    pub fn matvec2(
931        &self,
932        x1: &[f32],
933        x2: &[f32],
934        o1: &mut [f32],
935        o2: &mut [f32],
936        pool: Option<&Pool>,
937    ) {
938        match self {
939            Self::F32 { data, .. } => matvec_rows2(pool, data, x1, x2, o1, o2),
940            Self::Mapped {
941                dtype,
942                rows,
943                cols,
944                row_scale,
945                col_field,
946                vbit_offsets,
947                ..
948            } => {
949                if *dtype == TensorDtype::Q4Block {
950                    q4matvec2(self.quant_bytes(), x1, x2, *rows, *cols, o1, o2, pool);
951                    return;
952                }
953                if *dtype == TensorDtype::Q4Tiled {
954                    q4t_matvec2(self.quant_bytes(), x1, x2, *rows, *cols, o1, o2, pool);
955                    return;
956                }
957                if *dtype == TensorDtype::Q1 {
958                    q1_matvec2(self.quant_bytes(), x1, x2, *rows, *cols, o1, o2, pool);
959                    return;
960                }
961                if *dtype == TensorDtype::Q1T {
962                    // Fused ternary pair: one row pass, the register
963                    // unpack shared across both streams on ARM. (Q1T
964                    // lacks a row_scale array — scales live inline in
965                    // the tiles — so it must not fall through to the
966                    // q8 qmatvec2 below.)
967                    q1t_matvec2(self.quant_bytes(), x1, x2, *rows, *cols, o1, o2, pool);
968                    return;
969                }
970                if matches!(dtype, TensorDtype::Vbit | TensorDtype::VbitRo) {
971                    vbitmatvec2(
972                        self.quant_bytes(),
973                        vbit_offsets,
974                        x1,
975                        x2,
976                        *rows,
977                        *cols,
978                        o1,
979                        o2,
980                        pool,
981                    );
982                    return;
983                }
984                qmatvec2(
985                    self.quant_bytes(),
986                    row_scale,
987                    x1,
988                    x2,
989                    col_field,
990                    *dtype,
991                    *rows,
992                    *cols,
993                    o1,
994                    o2,
995                    pool,
996                );
997            }
998        }
999    }
1000}
1001
1002impl QTensor {
1003    /// Batched matvec (prefill-GEMM): xs — row-major [b, cols],
1004    /// out — row-major [b, rows]. Element-wise semantics are IDENTICAL
1005    /// to b matvec calls (same dot kernels in the same order); the win —
1006    /// the weight row streams from DRAM once per batch, not b times.
1007    pub fn matmat(&self, xs_all: &[f32], b: usize, out: &mut [f32], pool: Option<&Pool>) {
1008        let cols = self.cols();
1009        let rows = self.rows();
1010        debug_assert_eq!(xs_all.len(), b * cols);
1011        debug_assert_eq!(out.len(), b * rows);
1012        // GPTQ calibration: fold this layer's inputs into its Hessian. Only
1013        // Mapped tensors carry a directory name; the check is a relaxed
1014        // atomic load, free when not calibrating.
1015        if crate::gptq_capture::capturing() {
1016            if let Self::Mapped { model, idx, .. } = self {
1017                crate::gptq_capture::accumulate(&model.tensors[*idx].name, xs_all, b, cols);
1018            }
1019        }
1020        match self {
1021            Self::F32 { data, .. } => {
1022                let out_addr = SendMut(out.as_mut_ptr());
1023                let run = |start: usize, end: usize| {
1024                    for o in start..end {
1025                        let row = &data[o * cols..(o + 1) * cols];
1026                        for bi in 0..b {
1027                            let x = &xs_all[bi * cols..(bi + 1) * cols];
1028                            let mut acc = 0f32;
1029                            for j in 0..cols {
1030                                acc += row[j] * x[j];
1031                            }
1032                            unsafe { *out_addr.at(bi * rows + o) = acc };
1033                        }
1034                    }
1035                };
1036                dispatch_rows(pool, rows, &run);
1037            }
1038            Self::Mapped {
1039                dtype,
1040                row_scale,
1041                col_field,
1042                vbit_offsets,
1043                ..
1044            } => {
1045                if *dtype == TensorDtype::Q4Block {
1046                    q4matmat(self.quant_bytes(), xs_all, b, rows, cols, out, pool);
1047                    return;
1048                }
1049                if *dtype == TensorDtype::Q4Tiled {
1050                    // GPU batched q4t GEMM (dequant + f32nt mul_mm on the
1051                    // device); the probe keeps whichever beats the CPU arm.
1052                    // Narrow (prompt-encode) and wide (DiT) batches probe
1053                    // as separate classes — the regimes have opposite
1054                    // winners and one shared verdict locked the wrong arm.
1055                    // Kill switch (gpu::mm_kill): one grossly slow GPU op
1056                    // (a fair-condition op is ≤~100 ms even at 1024px)
1057                    // means the device is contended by another process
1058                    // (e.g. a simulator) — verdicts are per-process, so
1059                    // without the bail the whole render crawls behind
1060                    // someone else's queue.
1061                    if b >= 32
1062                        && b * rows * cols >= 128_000_000
1063                        && cols % 32 == 0
1064                        && !crate::gpu::mm_killed()
1065                        && crate::gpu::enabled_here()
1066                    {
1067                        let class = if b >= 128 {
1068                            crate::gpu::OpClass::MatmatWide
1069                        } else {
1070                            crate::gpu::OpClass::Matmat
1071                        };
1072                        if let Self::Mapped { model, idx, .. } = self {
1073                            let t0 = std::time::Instant::now();
1074                            match crate::gpu::probe_arm(class) {
1075                                crate::gpu::ProbeArm::Gpu => {
1076                                    if crate::gpu::q4t_matmat(
1077                                        model, *idx, xs_all, b, rows, cols, out,
1078                                    ) {
1079                                        let el = t0.elapsed();
1080                                        // Work-proportional budget: ~8× the
1081                                        // fair-device estimate (+20 ms slack).
1082                                        // An absolute cap missed the worst
1083                                        // case — contended ops sit at
1084                                        // 100–240 ms each and still bury a
1085                                        // render whose fair op is 3–9 ms.
1086                                        // Cold ops (first PSO build, buffer
1087                                        // alloc) are exempt: a one-off
1088                                        // ~50 ms compile is not contention.
1089                                        let flops = 2.0 * b as f64 * rows as f64 * cols as f64;
1090                                        let budget = std::time::Duration::from_secs_f64(
1091                                            flops / 1.5e12 * 8.0 + 0.020,
1092                                        );
1093                                        if el > budget && !crate::gpu::probe_was_cold() {
1094                                            tracing::warn!(
1095                                                "gpu q4t matmat took {el:?} (budget {budget:?}) — \
1096                                                 device contended, CPU for the rest of the process"
1097                                            );
1098                                            crate::gpu::mm_kill();
1099                                        }
1100                                        crate::gpu::probe_record(class, true, el);
1101                                        return;
1102                                    }
1103                                }
1104                                crate::gpu::ProbeArm::CpuTimed => {
1105                                    q4t_matmat(
1106                                        self.quant_bytes(),
1107                                        xs_all,
1108                                        b,
1109                                        rows,
1110                                        cols,
1111                                        out,
1112                                        pool,
1113                                    );
1114                                    crate::gpu::probe_record(class, false, t0.elapsed());
1115                                    return;
1116                                }
1117                                crate::gpu::ProbeArm::Cpu => {}
1118                            }
1119                        }
1120                    }
1121                    q4t_matmat(self.quant_bytes(), xs_all, b, rows, cols, out, pool);
1122                    return;
1123                }
1124                if *dtype == TensorDtype::Q1 {
1125                    // GPU batched q1 GEMM for wide prefill (q1_mul_mm on the
1126                    // device); the probe keeps whichever beats the CPU matmat.
1127                    if b >= 32
1128                        && b * rows * cols >= 128_000_000
1129                        && cols % 64 == 0
1130                        && crate::gpu::enabled_here()
1131                    {
1132                        if let Self::Mapped { model, idx, .. } = self {
1133                            let t0 = std::time::Instant::now();
1134                            match crate::gpu::probe_arm(crate::gpu::OpClass::Matmat) {
1135                                crate::gpu::ProbeArm::Gpu => {
1136                                    if crate::gpu::q1_matmat(
1137                                        model, *idx, xs_all, b, rows, cols, out,
1138                                    ) {
1139                                        crate::gpu::probe_record(
1140                                            crate::gpu::OpClass::Matmat,
1141                                            true,
1142                                            t0.elapsed(),
1143                                        );
1144                                        return;
1145                                    }
1146                                }
1147                                crate::gpu::ProbeArm::CpuTimed => {
1148                                    q1_matmat(self.quant_bytes(), xs_all, b, rows, cols, out, pool);
1149                                    crate::gpu::probe_record(
1150                                        crate::gpu::OpClass::Matmat,
1151                                        false,
1152                                        t0.elapsed(),
1153                                    );
1154                                    return;
1155                                }
1156                                crate::gpu::ProbeArm::Cpu => {}
1157                            }
1158                        }
1159                    }
1160                    q1_matmat(self.quant_bytes(), xs_all, b, rows, cols, out, pool);
1161                    return;
1162                }
1163                if *dtype == TensorDtype::Q1T {
1164                    // GPU batched GEMM for wide prefill (base + overlay on the
1165                    // device); probe keeps the winner vs the CPU matmat.
1166                    if b >= 32 && b * rows * cols >= 128_000_000 && crate::gpu::enabled_here() {
1167                        if let Self::Mapped { model, idx, .. } = self {
1168                            let t0 = std::time::Instant::now();
1169                            match crate::gpu::probe_arm(crate::gpu::OpClass::Matmat) {
1170                                crate::gpu::ProbeArm::Gpu => {
1171                                    if crate::gpu::q1t_matmat(
1172                                        model, *idx, xs_all, b, rows, cols, out,
1173                                    ) {
1174                                        crate::gpu::probe_record(
1175                                            crate::gpu::OpClass::Matmat,
1176                                            true,
1177                                            t0.elapsed(),
1178                                        );
1179                                        return;
1180                                    }
1181                                }
1182                                crate::gpu::ProbeArm::CpuTimed => {
1183                                    q1t_matmat(
1184                                        self.quant_bytes(),
1185                                        xs_all,
1186                                        b,
1187                                        rows,
1188                                        cols,
1189                                        out,
1190                                        pool,
1191                                    );
1192                                    crate::gpu::probe_record(
1193                                        crate::gpu::OpClass::Matmat,
1194                                        false,
1195                                        t0.elapsed(),
1196                                    );
1197                                    return;
1198                                }
1199                                crate::gpu::ProbeArm::Cpu => {}
1200                            }
1201                        }
1202                    }
1203                    q1t_matmat(self.quant_bytes(), xs_all, b, rows, cols, out, pool);
1204                    return;
1205                }
1206                if matches!(dtype, TensorDtype::Vbit | TensorDtype::VbitRo) {
1207                    vbitmatmat(
1208                        self.quant_bytes(),
1209                        vbit_offsets,
1210                        xs_all,
1211                        b,
1212                        rows,
1213                        cols,
1214                        out,
1215                        pool,
1216                    );
1217                    return;
1218                }
1219                let pre: Vec<std::borrow::Cow<'_, [f32]>> = (0..b)
1220                    .map(|bi| prescale(&xs_all[bi * cols..(bi + 1) * cols], col_field, *dtype))
1221                    .collect();
1222                // D5: large prefill-batch GEMMs — on the GPU (threshold by
1223                // work volume: submission carries b×rows×cols MACs).
1224                // Runtime probe: the naive GEMM shader + sync readback
1225                // lose to the CPU GEMM on slow driver stacks — alternate
1226                // both arms and keep the winner.
1227                if b >= 8 && b * rows * cols >= 128_000_000 && crate::gpu::enabled_here() {
1228                    if let Self::Mapped { model, idx, .. } = self {
1229                        let t0 = std::time::Instant::now();
1230                        match crate::gpu::probe_arm(crate::gpu::OpClass::Matmat) {
1231                            crate::gpu::ProbeArm::Gpu
1232                                if crate::gpu::probe_deciding(crate::gpu::OpClass::Matmat)
1233                                    && !crate::gpu::q8_resident_or_upload(model, *idx) =>
1234                            {
1235                                // Cold weights during probing: the upload
1236                                // has started, the count runs on the CPU —
1237                                // the GPU arm samples on the next touch.
1238                                let q = self.quant_bytes();
1239                                qmatmat(q, row_scale, &pre, rows, cols, out, pool);
1240                                return;
1241                            }
1242                            crate::gpu::ProbeArm::Gpu => {
1243                                let flat: Vec<f32> =
1244                                    pre.iter().flat_map(|v| v.iter().copied()).collect();
1245                                if crate::gpu::q8_matmat(
1246                                    model, *idx, row_scale, &flat, b, rows, cols, out,
1247                                ) {
1248                                    crate::gpu::probe_record(
1249                                        crate::gpu::OpClass::Matmat,
1250                                        true,
1251                                        t0.elapsed(),
1252                                    );
1253                                    return;
1254                                }
1255                            }
1256                            crate::gpu::ProbeArm::CpuTimed => {
1257                                let q = self.quant_bytes();
1258                                qmatmat(q, row_scale, &pre, rows, cols, out, pool);
1259                                crate::gpu::probe_record(
1260                                    crate::gpu::OpClass::Matmat,
1261                                    false,
1262                                    t0.elapsed(),
1263                                );
1264                                return;
1265                            }
1266                            crate::gpu::ProbeArm::Cpu => {}
1267                        }
1268                    }
1269                }
1270                let q = self.quant_bytes();
1271                qmatmat(q, row_scale, &pre, rows, cols, out, pool);
1272            }
1273        }
1274    }
1275}
1276
1277impl QTensor {
1278    /// Multi-matrix job (roadmap §3 P0): N tensors sharing one input
1279    /// run under a SINGLE pool dispatch — QKV or gate+up cost one
1280    /// barrier instead of N. Per-row math is the exact same kernel as
1281    /// `matvec` (bit-identical outputs); only the dispatch is fused.
1282    /// Falls back to N sequential matvecs when the set is not a uniform
1283    /// q8-family/F32 group or there is no pool.
1284    pub fn matvec_many<const N: usize>(
1285        ts: [&QTensor; N],
1286        x: &[f32],
1287        mut outs: [&mut [f32]; N],
1288        pool: Option<&Pool>,
1289    ) {
1290        let total_rows: usize = ts.iter().map(|t| t.rows()).sum();
1291        let uniform_q8 = ts.iter().all(|t| {
1292            matches!(
1293                t,
1294                Self::Mapped {
1295                    dtype: TensorDtype::Q8Row | TensorDtype::Q8_2f,
1296                    ..
1297                }
1298            )
1299        });
1300        let uniform_f32 = ts.iter().all(|t| matches!(t, Self::F32 { .. }));
1301        let uniform_q4 = ts.iter().all(|t| {
1302            matches!(
1303                t,
1304                Self::Mapped {
1305                    dtype: TensorDtype::Q4Block,
1306                    ..
1307                }
1308            )
1309        });
1310        let uniform_vbit = ts.iter().all(|t| {
1311            matches!(
1312                t,
1313                Self::Mapped {
1314                    dtype: TensorDtype::Vbit | TensorDtype::VbitRo,
1315                    ..
1316                }
1317            )
1318        });
1319        let uniform_q1 = ts.iter().all(|t| {
1320            matches!(
1321                t,
1322                Self::Mapped {
1323                    dtype: TensorDtype::Q1,
1324                    ..
1325                }
1326            )
1327        });
1328        let uniform_q1t = ts.iter().all(|t| {
1329            matches!(
1330                t,
1331                Self::Mapped {
1332                    dtype: TensorDtype::Q1T,
1333                    ..
1334                }
1335            )
1336        });
1337        let Some(pool) = pool else {
1338            for (t, o) in ts.iter().zip(outs.iter_mut()) {
1339                t.matvec(x, o, None);
1340            }
1341            return;
1342        };
1343        if total_rows < 256
1344            || !(uniform_q8
1345                || uniform_f32
1346                || uniform_q4
1347                || uniform_vbit
1348                || uniform_q1
1349                || uniform_q1t)
1350        {
1351            for (t, o) in ts.iter().zip(outs.iter_mut()) {
1352                t.matvec(x, o, Some(pool));
1353            }
1354            return;
1355        }
1356
1357        if uniform_q1 {
1358            // One shared activation split + group sums (q1 has no col
1359            // field; the same input feeds every tensor).
1360            let outs_addr: [SendMut; N] = std::array::from_fn(|i| SendMut(outs[i].as_mut_ptr()));
1361            if a8w8_enabled() {
1362                let act = split_act(x);
1363                let gsum = q1_group_sums(&act.xq, ts[0].cols() / GROUP_SIZE);
1364                let (act, gsum) = (&act, &gsum);
1365                let closures: [_; N] = std::array::from_fn(|i| {
1366                    let (bytes, gpr, out) =
1367                        (ts[i].quant_bytes(), ts[i].cols() / GROUP_SIZE, outs_addr[i]);
1368                    move |s: usize, e: usize| q1_range_a8w8(bytes, gpr, act, gsum, out, s, e)
1369                });
1370                let parts: [(usize, &(dyn Fn(usize, usize) + Sync)); N] =
1371                    std::array::from_fn(|i| (ts[i].rows(), &closures[i] as _));
1372                pool.run_many(&parts);
1373            } else {
1374                let closures: [_; N] = std::array::from_fn(|i| {
1375                    let (bytes, gpr, out) =
1376                        (ts[i].quant_bytes(), ts[i].cols() / GROUP_SIZE, outs_addr[i]);
1377                    move |s: usize, e: usize| q1_range_f32(bytes, gpr, x, out, s, e)
1378                });
1379                let parts: [(usize, &(dyn Fn(usize, usize) + Sync)); N] =
1380                    std::array::from_fn(|i| (ts[i].rows(), &closures[i] as _));
1381                pool.run_many(&parts);
1382            }
1383            return;
1384        }
1385
1386        if uniform_q1t {
1387            // Q1T batched: one shared activation split + overlay decode,
1388            // all tensors' rows in ONE pool dispatch (saves N−1 dispatches
1389            // and N−1 redundant split_act calls per layer).
1390            let outs_addr: [SendMut; N] = std::array::from_fn(|i| SendMut(outs[i].as_mut_ptr()));
1391            const TILE: usize = cortiq_core::quant::Q1T_TILE;
1392            if a8w8_enabled() {
1393                let act = split_act(x);
1394                let act = &act;
1395                let x_ref = x;
1396                let closures: [_; N] = std::array::from_fn(|i| {
1397                    let bytes = ts[i].quant_bytes();
1398                    let (rows, cols) = (ts[i].rows(), ts[i].cols());
1399                    let gpr = cols / GROUP_SIZE;
1400                    let (rp_off, ent_off, has_ov) = q1t_overlay(bytes, rows * gpr * TILE, rows);
1401                    let out = outs_addr[i];
1402                    move |s: usize, e: usize| {
1403                        q1t_range_a8w8(bytes, gpr, rp_off, ent_off, has_ov, act, x_ref, out, s, e)
1404                    }
1405                });
1406                let parts: [(usize, &(dyn Fn(usize, usize) + Sync)); N] =
1407                    std::array::from_fn(|i| (ts[i].rows(), &closures[i] as _));
1408                pool.run_many(&parts);
1409            } else {
1410                let x_ref = x;
1411                let closures: [_; N] = std::array::from_fn(|i| {
1412                    let bytes = ts[i].quant_bytes();
1413                    let (rows, cols) = (ts[i].rows(), ts[i].cols());
1414                    let gpr = cols / GROUP_SIZE;
1415                    let (rp_off, ent_off, has_ov) = q1t_overlay(bytes, rows * gpr * TILE, rows);
1416                    let out = outs_addr[i];
1417                    move |s: usize, e: usize| {
1418                        q1t_range_f32_batch(bytes, gpr, rp_off, ent_off, has_ov, x_ref, out, s, e)
1419                    }
1420                });
1421                let parts: [(usize, &(dyn Fn(usize, usize) + Sync)); N] =
1422                    std::array::from_fn(|i| (ts[i].rows(), &closures[i] as _));
1423                pool.run_many(&parts);
1424            }
1425            return;
1426        }
1427
1428        if uniform_q4 || uniform_vbit {
1429            let outs_addr: [SendMut; N] = std::array::from_fn(|i| SendMut(outs[i].as_mut_ptr()));
1430            // q4/vbit share one activation split — no per-tensor col field.
1431            if a8w8_enabled() {
1432                let act = split_act(x);
1433                let act = &act;
1434                if uniform_q4 {
1435                    let closures: [_; N] = std::array::from_fn(|i| {
1436                        let (packed, scales) =
1437                            q4_split(ts[i].quant_bytes(), ts[i].rows(), ts[i].cols());
1438                        let (gpr, cols, out) =
1439                            (ts[i].cols() / GROUP_SIZE, ts[i].cols(), outs_addr[i]);
1440                        move |s: usize, e: usize| {
1441                            q4_range_a8w8(packed, scales, gpr, cols, act, out, s, e)
1442                        }
1443                    });
1444                    let parts: [(usize, &(dyn Fn(usize, usize) + Sync)); N] =
1445                        std::array::from_fn(|i| (ts[i].rows(), &closures[i] as _));
1446                    pool.run_many(&parts);
1447                } else {
1448                    let closures: [_; N] = std::array::from_fn(|i| {
1449                        let Self::Mapped { vbit_offsets, .. } = ts[i] else {
1450                            unreachable!()
1451                        };
1452                        let (bytes, rows, cols, out) = (
1453                            ts[i].quant_bytes(),
1454                            ts[i].rows(),
1455                            ts[i].cols(),
1456                            outs_addr[i],
1457                        );
1458                        move |s: usize, e: usize| {
1459                            vbit_range_a8w8(bytes, vbit_offsets, x, act, rows, cols, out, s, e)
1460                        }
1461                    });
1462                    let parts: [(usize, &(dyn Fn(usize, usize) + Sync)); N] =
1463                        std::array::from_fn(|i| (ts[i].rows(), &closures[i] as _));
1464                    pool.run_many(&parts);
1465                }
1466                return;
1467            }
1468            if uniform_q4 {
1469                let closures: [_; N] = std::array::from_fn(|i| {
1470                    let (packed, scales) =
1471                        q4_split(ts[i].quant_bytes(), ts[i].rows(), ts[i].cols());
1472                    let (gpr, out) = (ts[i].cols() / GROUP_SIZE, outs_addr[i]);
1473                    move |s: usize, e: usize| q4_range_f32(packed, scales, gpr, x, out, s, e)
1474                });
1475                let parts: [(usize, &(dyn Fn(usize, usize) + Sync)); N] =
1476                    std::array::from_fn(|i| (ts[i].rows(), &closures[i] as _));
1477                pool.run_many(&parts);
1478            } else {
1479                let closures: [_; N] = std::array::from_fn(|i| {
1480                    let Self::Mapped { vbit_offsets, .. } = ts[i] else {
1481                        unreachable!()
1482                    };
1483                    let (bytes, rows, cols, out) = (
1484                        ts[i].quant_bytes(),
1485                        ts[i].rows(),
1486                        ts[i].cols(),
1487                        outs_addr[i],
1488                    );
1489                    move |s: usize, e: usize| {
1490                        vbit_range_f32(bytes, vbit_offsets, x, rows, cols, out, s, e)
1491                    }
1492                });
1493                let parts: [(usize, &(dyn Fn(usize, usize) + Sync)); N] =
1494                    std::array::from_fn(|i| (ts[i].rows(), &closures[i] as _));
1495                pool.run_many(&parts);
1496            }
1497            return;
1498        }
1499
1500        if uniform_f32 {
1501            let outs_addr: [SendMut; N] = std::array::from_fn(|i| SendMut(outs[i].as_mut_ptr()));
1502            let closures: [_; N] = std::array::from_fn(|i| {
1503                let Self::F32 { data, cols, .. } = ts[i] else {
1504                    unreachable!()
1505                };
1506                let out = outs_addr[i];
1507                move |start: usize, end: usize| {
1508                    for o in start..end {
1509                        let row = &data[o * cols..(o + 1) * cols];
1510                        let mut sum = 0.0f32;
1511                        for j in 0..*cols {
1512                            sum += row[j] * x[j];
1513                        }
1514                        // SAFETY: disjoint (tensor, row) cells per worker.
1515                        unsafe { *out.at(o) = sum };
1516                    }
1517                }
1518            });
1519            let parts: [(usize, &(dyn Fn(usize, usize) + Sync)); N] =
1520                std::array::from_fn(|i| (ts[i].rows(), &closures[i] as _));
1521            pool.run_many(&parts);
1522            return;
1523        }
1524
1525        // Uniform q8-family: per-tensor prescale (q8_2f col fields
1526        // differ per tensor) + the shared range kernels.
1527        struct Ctx<'a> {
1528            bytes: &'a [u8],
1529            #[cfg_attr(not(target_arch = "aarch64"), allow(dead_code))]
1530            rep: &'a [u8],
1531            row_scale: &'a [f32],
1532            cols: usize,
1533            xs: std::borrow::Cow<'a, [f32]>,
1534        }
1535        let ctxs: [Ctx<'_>; N] = std::array::from_fn(|i| {
1536            let Self::Mapped {
1537                dtype,
1538                cols,
1539                row_scale,
1540                col_field,
1541                repack,
1542                ..
1543            } = ts[i]
1544            else {
1545                unreachable!()
1546            };
1547            Ctx {
1548                bytes: ts[i].quant_bytes(),
1549                rep: repack,
1550                row_scale,
1551                cols: *cols,
1552                xs: prescale(x, col_field, *dtype),
1553            }
1554        });
1555        let outs_addr: [SendMut; N] = std::array::from_fn(|i| SendMut(outs[i].as_mut_ptr()));
1556        #[cfg(target_arch = "aarch64")]
1557        if sdot_enabled() {
1558            let acts: [SplitAct; N] = std::array::from_fn(|i| split_act(&ctxs[i].xs));
1559            let closures: [_; N] = std::array::from_fn(|i| {
1560                let (c, act, out) = (&ctxs[i], &acts[i], outs_addr[i]);
1561                move |start: usize, end: usize| {
1562                    q8_range_sdot(c.bytes, c.rep, c.row_scale, act, c.cols, out, start, end)
1563                }
1564            });
1565            let parts: [(usize, &(dyn Fn(usize, usize) + Sync)); N] =
1566                std::array::from_fn(|i| (ts[i].rows(), &closures[i] as _));
1567            pool.run_many(&parts);
1568            return;
1569        }
1570        #[cfg(target_arch = "x86_64")]
1571        if avx2_a8w8_enabled() {
1572            let acts: [SplitAct; N] = std::array::from_fn(|i| split_act(&ctxs[i].xs));
1573            let closures: [_; N] = std::array::from_fn(|i| {
1574                let (c, act, out) = (&ctxs[i], &acts[i], outs_addr[i]);
1575                move |start: usize, end: usize| {
1576                    q8_range_avx2(c.bytes, c.row_scale, act, c.cols, out, start, end)
1577                }
1578            });
1579            let parts: [(usize, &(dyn Fn(usize, usize) + Sync)); N] =
1580                std::array::from_fn(|i| (ts[i].rows(), &closures[i] as _));
1581            pool.run_many(&parts);
1582            return;
1583        }
1584        let closures: [_; N] = std::array::from_fn(|i| {
1585            let (c, out) = (&ctxs[i], outs_addr[i]);
1586            move |start: usize, end: usize| {
1587                q8_range_f32(c.bytes, c.row_scale, &c.xs, c.cols, out, start, end)
1588            }
1589        });
1590        let parts: [(usize, &(dyn Fn(usize, usize) + Sync)); N] =
1591            std::array::from_fn(|i| (ts[i].rows(), &closures[i] as _));
1592        pool.run_many(&parts);
1593    }
1594}
1595
1596impl QTensor {
1597    /// Pair-input multi-matrix job: N tensors × 2 shared inputs under a
1598    /// single pool dispatch — the MTP/pair decode path publishes one job
1599    /// for Q/K/V (and one for gate+up) instead of one per tensor.
1600    /// Per-row math is exactly `matvec2`'s kernels; bit-identical.
1601    #[allow(clippy::needless_range_loop)]
1602    pub fn matvec2_many<const N: usize>(
1603        ts: [&QTensor; N],
1604        x1: &[f32],
1605        x2: &[f32],
1606        mut o1s: [&mut [f32]; N],
1607        mut o2s: [&mut [f32]; N],
1608        pool: Option<&Pool>,
1609    ) {
1610        let total_rows: usize = ts.iter().map(|t| t.rows()).sum();
1611        let uniform_q8 = ts.iter().all(|t| {
1612            matches!(
1613                t,
1614                Self::Mapped {
1615                    dtype: TensorDtype::Q8Row | TensorDtype::Q8_2f,
1616                    ..
1617                }
1618            )
1619        });
1620        let uniform_f32 = ts.iter().all(|t| matches!(t, Self::F32 { .. }));
1621        let uniform_q4 = ts.iter().all(|t| {
1622            matches!(
1623                t,
1624                Self::Mapped {
1625                    dtype: TensorDtype::Q4Block,
1626                    ..
1627                }
1628            )
1629        });
1630        let uniform_vbit = ts.iter().all(|t| {
1631            matches!(
1632                t,
1633                Self::Mapped {
1634                    dtype: TensorDtype::Vbit | TensorDtype::VbitRo,
1635                    ..
1636                }
1637            )
1638        });
1639        let fusable = pool.is_some()
1640            && total_rows >= 256
1641            && (uniform_q8 || uniform_f32 || uniform_q4 || uniform_vbit);
1642        if !fusable {
1643            for i in 0..N {
1644                ts[i].matvec2(x1, x2, o1s[i], o2s[i], pool);
1645            }
1646            return;
1647        }
1648        let pool = pool.unwrap();
1649
1650        if uniform_q4 || uniform_vbit {
1651            let p1: [SendMut; N] = std::array::from_fn(|i| SendMut(o1s[i].as_mut_ptr()));
1652            let p2: [SendMut; N] = std::array::from_fn(|i| SendMut(o2s[i].as_mut_ptr()));
1653            // q4/vbit share activation splits — no per-tensor col field.
1654            if a8w8_enabled() {
1655                let a1 = split_act(x1);
1656                let a2 = split_act(x2);
1657                let (a1, a2) = (&a1, &a2);
1658                if uniform_q4 {
1659                    let closures: [_; N] = std::array::from_fn(|i| {
1660                        let (packed, scales) =
1661                            q4_split(ts[i].quant_bytes(), ts[i].rows(), ts[i].cols());
1662                        let (gpr, cols, o1, o2) =
1663                            (ts[i].cols() / GROUP_SIZE, ts[i].cols(), p1[i], p2[i]);
1664                        move |s: usize, e: usize| {
1665                            q4_range2_a8w8(packed, scales, gpr, cols, a1, a2, o1, o2, s, e)
1666                        }
1667                    });
1668                    let parts: [(usize, &(dyn Fn(usize, usize) + Sync)); N] =
1669                        std::array::from_fn(|i| (ts[i].rows(), &closures[i] as _));
1670                    pool.run_many(&parts);
1671                } else {
1672                    let closures: [_; N] = std::array::from_fn(|i| {
1673                        let Self::Mapped { vbit_offsets, .. } = ts[i] else {
1674                            unreachable!()
1675                        };
1676                        let (bytes, rows, cols, o1, o2) = (
1677                            ts[i].quant_bytes(),
1678                            ts[i].rows(),
1679                            ts[i].cols(),
1680                            p1[i],
1681                            p2[i],
1682                        );
1683                        move |s: usize, e: usize| {
1684                            vbit_range2_a8w8(
1685                                bytes,
1686                                vbit_offsets,
1687                                x1,
1688                                x2,
1689                                a1,
1690                                a2,
1691                                rows,
1692                                cols,
1693                                o1,
1694                                o2,
1695                                s,
1696                                e,
1697                            )
1698                        }
1699                    });
1700                    let parts: [(usize, &(dyn Fn(usize, usize) + Sync)); N] =
1701                        std::array::from_fn(|i| (ts[i].rows(), &closures[i] as _));
1702                    pool.run_many(&parts);
1703                }
1704                return;
1705            }
1706            if uniform_q4 {
1707                let closures: [_; N] = std::array::from_fn(|i| {
1708                    let (packed, scales) =
1709                        q4_split(ts[i].quant_bytes(), ts[i].rows(), ts[i].cols());
1710                    let (gpr, o1, o2) = (ts[i].cols() / GROUP_SIZE, p1[i], p2[i]);
1711                    move |s: usize, e: usize| {
1712                        q4_range2_f32(packed, scales, gpr, x1, x2, o1, o2, s, e)
1713                    }
1714                });
1715                let parts: [(usize, &(dyn Fn(usize, usize) + Sync)); N] =
1716                    std::array::from_fn(|i| (ts[i].rows(), &closures[i] as _));
1717                pool.run_many(&parts);
1718            } else {
1719                let closures: [_; N] = std::array::from_fn(|i| {
1720                    let Self::Mapped { vbit_offsets, .. } = ts[i] else {
1721                        unreachable!()
1722                    };
1723                    let (bytes, rows, cols, o1, o2) = (
1724                        ts[i].quant_bytes(),
1725                        ts[i].rows(),
1726                        ts[i].cols(),
1727                        p1[i],
1728                        p2[i],
1729                    );
1730                    move |s: usize, e: usize| {
1731                        vbit_range2_f32(bytes, vbit_offsets, x1, x2, rows, cols, o1, o2, s, e)
1732                    }
1733                });
1734                let parts: [(usize, &(dyn Fn(usize, usize) + Sync)); N] =
1735                    std::array::from_fn(|i| (ts[i].rows(), &closures[i] as _));
1736                pool.run_many(&parts);
1737            }
1738            return;
1739        }
1740
1741        if uniform_f32 {
1742            let p1: [SendMut; N] = std::array::from_fn(|i| SendMut(o1s[i].as_mut_ptr()));
1743            let p2: [SendMut; N] = std::array::from_fn(|i| SendMut(o2s[i].as_mut_ptr()));
1744            let closures: [_; N] = std::array::from_fn(|i| {
1745                let Self::F32 { data, cols, .. } = ts[i] else {
1746                    unreachable!()
1747                };
1748                let (o1, o2) = (p1[i], p2[i]);
1749                move |start: usize, end: usize| {
1750                    for o in start..end {
1751                        let row = &data[o * cols..(o + 1) * cols];
1752                        let (mut s1, mut s2) = (0.0f32, 0.0f32);
1753                        for j in 0..*cols {
1754                            s1 += row[j] * x1[j];
1755                            s2 += row[j] * x2[j];
1756                        }
1757                        // SAFETY: disjoint (tensor, row) cells per worker.
1758                        unsafe {
1759                            *o1.at(o) = s1;
1760                            *o2.at(o) = s2;
1761                        }
1762                    }
1763                }
1764            });
1765            let parts: [(usize, &(dyn Fn(usize, usize) + Sync)); N] =
1766                std::array::from_fn(|i| (ts[i].rows(), &closures[i] as _));
1767            pool.run_many(&parts);
1768            return;
1769        }
1770
1771        struct Ctx<'a> {
1772            bytes: &'a [u8],
1773            row_scale: &'a [f32],
1774            cols: usize,
1775            xs1: std::borrow::Cow<'a, [f32]>,
1776            xs2: std::borrow::Cow<'a, [f32]>,
1777        }
1778        let ctxs: [Ctx<'_>; N] = std::array::from_fn(|i| {
1779            let Self::Mapped {
1780                dtype,
1781                cols,
1782                row_scale,
1783                col_field,
1784                ..
1785            } = ts[i]
1786            else {
1787                unreachable!()
1788            };
1789            Ctx {
1790                bytes: ts[i].quant_bytes(),
1791                row_scale,
1792                cols: *cols,
1793                xs1: prescale(x1, col_field, *dtype),
1794                xs2: prescale(x2, col_field, *dtype),
1795            }
1796        });
1797        let p1: [SendMut; N] = std::array::from_fn(|i| SendMut(o1s[i].as_mut_ptr()));
1798        let p2: [SendMut; N] = std::array::from_fn(|i| SendMut(o2s[i].as_mut_ptr()));
1799        #[cfg(target_arch = "aarch64")]
1800        if sdot_enabled() {
1801            let acts: [(SplitAct, SplitAct); N] =
1802                std::array::from_fn(|i| (split_act(&ctxs[i].xs1), split_act(&ctxs[i].xs2)));
1803            let closures: [_; N] = std::array::from_fn(|i| {
1804                let (c, a, o1, o2) = (&ctxs[i], &acts[i], p1[i], p2[i]);
1805                move |start: usize, end: usize| {
1806                    q8_range2_sdot(c.bytes, c.row_scale, &a.0, &a.1, c.cols, o1, o2, start, end)
1807                }
1808            });
1809            let parts: [(usize, &(dyn Fn(usize, usize) + Sync)); N] =
1810                std::array::from_fn(|i| (ts[i].rows(), &closures[i] as _));
1811            pool.run_many(&parts);
1812            return;
1813        }
1814        #[cfg(target_arch = "x86_64")]
1815        if avx2_a8w8_enabled() {
1816            let acts: [(SplitAct, SplitAct); N] =
1817                std::array::from_fn(|i| (split_act(&ctxs[i].xs1), split_act(&ctxs[i].xs2)));
1818            let closures: [_; N] = std::array::from_fn(|i| {
1819                let (c, a, o1, o2) = (&ctxs[i], &acts[i], p1[i], p2[i]);
1820                move |start: usize, end: usize| {
1821                    q8_range2_avx2(c.bytes, c.row_scale, &a.0, &a.1, c.cols, o1, o2, start, end)
1822                }
1823            });
1824            let parts: [(usize, &(dyn Fn(usize, usize) + Sync)); N] =
1825                std::array::from_fn(|i| (ts[i].rows(), &closures[i] as _));
1826            pool.run_many(&parts);
1827            return;
1828        }
1829        let closures: [_; N] = std::array::from_fn(|i| {
1830            let (c, o1, o2) = (&ctxs[i], p1[i], p2[i]);
1831            move |start: usize, end: usize| {
1832                q8_range2_f32(
1833                    c.bytes,
1834                    c.row_scale,
1835                    &c.xs1,
1836                    &c.xs2,
1837                    c.cols,
1838                    o1,
1839                    o2,
1840                    start,
1841                    end,
1842                )
1843            }
1844        });
1845        let parts: [(usize, &(dyn Fn(usize, usize) + Sync)); N] =
1846            std::array::from_fn(|i| (ts[i].rows(), &closures[i] as _));
1847        pool.run_many(&parts);
1848    }
1849
1850    /// Fused gate+up matvec with SiLU·mul: for each row r, computes
1851    /// `silu(gate·x) * (up·x)` and writes to `out[r]`. ONE pool dispatch,
1852    /// no intermediate g/u buffers, no separate silu pass. Falls back
1853    /// (returns false) for unsupported dtype combos.
1854    pub fn matvec_silu_mul(
1855        gate: &QTensor,
1856        up: &QTensor,
1857        x: &[f32],
1858        out: &mut [f32],
1859        pool: Option<&Pool>,
1860    ) -> bool {
1861        let inter = gate.rows();
1862        debug_assert_eq!(up.rows(), inter);
1863        debug_assert_eq!(out.len(), inter);
1864        debug_assert_eq!(gate.cols(), up.cols());
1865        if !a8w8_enabled() {
1866            return false;
1867        }
1868        let act = split_act(x);
1869        let act = &act;
1870        let x_ref = x;
1871        let out_addr = SendMut(out.as_mut_ptr());
1872
1873        match (gate, up) {
1874            // Q4Block gate + Q4Block up (most common mobile q4 models)
1875            (
1876                Self::Mapped {
1877                    dtype: TensorDtype::Q4Block,
1878                    ..
1879                },
1880                Self::Mapped {
1881                    dtype: TensorDtype::Q4Block,
1882                    ..
1883                },
1884            ) => {
1885                let (gp, gs) = q4_split(gate.quant_bytes(), gate.rows(), gate.cols());
1886                let (up_p, up_s) = q4_split(up.quant_bytes(), up.rows(), up.cols());
1887                let gpr = gate.cols() / GROUP_SIZE;
1888                let cols = gate.cols();
1889                let run = move |start: usize, end: usize| {
1890                    for r in start..end {
1891                        let mut gv = dot_q4_row_i8(gp, gs, r * gpr, gpr, &act.xq) * act.sx;
1892                        let mut uv = dot_q4_row_i8(up_p, up_s, r * gpr, gpr, &act.xq) * act.sx;
1893                        for &(j, xv) in &act.outliers {
1894                            let flat = r * cols + j;
1895                            let gb = gp[flat / 2];
1896                            let gn = if flat & 1 == 0 { gb & 0x0F } else { gb >> 4 };
1897                            let gsc = f16_to_f32(u16::from_le_bytes([
1898                                gs[(flat / GROUP_SIZE) * 2],
1899                                gs[(flat / GROUP_SIZE) * 2 + 1],
1900                            ]));
1901                            gv += ((gn as i32 - 8) as f32) * gsc * xv;
1902                            let ub = up_p[flat / 2];
1903                            let un = if flat & 1 == 0 { ub & 0x0F } else { ub >> 4 };
1904                            let usc = f16_to_f32(u16::from_le_bytes([
1905                                up_s[(flat / GROUP_SIZE) * 2],
1906                                up_s[(flat / GROUP_SIZE) * 2 + 1],
1907                            ]));
1908                            uv += ((un as i32 - 8) as f32) * usc * xv;
1909                        }
1910                        let silu_g = gv / (1.0 + (-gv).exp());
1911                        // SAFETY: disjoint row ranges per worker.
1912                        unsafe { *out_addr.at(r) = silu_g * uv };
1913                    }
1914                };
1915                dispatch_rows(pool, inter, &run);
1916                true
1917            }
1918            // Q4Tiled gate + Q4Tiled up — one row pass, both tile
1919            // streams sequential, silu·mul fused (same per-row math as
1920            // `q4t_matvec`).
1921            (
1922                Self::Mapped {
1923                    dtype: TensorDtype::Q4Tiled,
1924                    ..
1925                },
1926                Self::Mapped {
1927                    dtype: TensorDtype::Q4Tiled,
1928                    ..
1929                },
1930            ) => {
1931                let g_bytes = gate.quant_bytes();
1932                let u_bytes = up.quant_bytes();
1933                let gpr = gate.cols() / GROUP_SIZE;
1934                let run = move |start: usize, end: usize| {
1935                    for r in start..end {
1936                        let mut gv = dot_q4t_row_i8(g_bytes, r, gpr, &act.xq) * act.sx;
1937                        let mut uv = dot_q4t_row_i8(u_bytes, r, gpr, &act.xq) * act.sx;
1938                        for &(j, xv) in &act.outliers {
1939                            let (w, s) = q4t_outlier(g_bytes, r, gpr, j);
1940                            gv += w * s * xv;
1941                            let (w, s) = q4t_outlier(u_bytes, r, gpr, j);
1942                            uv += w * s * xv;
1943                        }
1944                        let silu_g = gv / (1.0 + (-gv).exp());
1945                        // SAFETY: disjoint row ranges per worker.
1946                        unsafe { *out_addr.at(r) = silu_g * uv };
1947                    }
1948                };
1949                dispatch_rows(pool, inter, &run);
1950                true
1951            }
1952            // Q1T gate + Q1T up
1953            (
1954                Self::Mapped {
1955                    dtype: TensorDtype::Q1T,
1956                    ..
1957                },
1958                Self::Mapped {
1959                    dtype: TensorDtype::Q1T,
1960                    ..
1961                },
1962            ) => {
1963                const TILE: usize = cortiq_core::quant::Q1T_TILE;
1964                let g_bytes = gate.quant_bytes();
1965                let u_bytes = up.quant_bytes();
1966                let gpr = gate.cols() / GROUP_SIZE;
1967                let (g_rp, g_ent, g_ov) = q1t_overlay(g_bytes, inter * gpr * TILE, inter);
1968                let (u_rp, u_ent, u_ov) = q1t_overlay(u_bytes, inter * gpr * TILE, inter);
1969                let run = move |start: usize, end: usize| {
1970                    for r in start..end {
1971                        let mut gv = q1t_dot_row_i8(g_bytes, r, gpr, &act.xq) * act.sx;
1972                        let mut uv = q1t_dot_row_i8(u_bytes, r, gpr, &act.xq) * act.sx;
1973                        for &(j, xv) in &act.outliers {
1974                            gv += q1t_base_weight(g_bytes, r, gpr, j) * xv;
1975                            uv += q1t_base_weight(u_bytes, r, gpr, j) * xv;
1976                        }
1977                        gv += q1t_row_outlier_correction(g_bytes, r, g_rp, g_ent, g_ov, x_ref);
1978                        uv += q1t_row_outlier_correction(u_bytes, r, u_rp, u_ent, u_ov, x_ref);
1979                        let silu_g = gv / (1.0 + (-gv).exp());
1980                        // SAFETY: disjoint row ranges per worker.
1981                        unsafe { *out_addr.at(r) = silu_g * uv };
1982                    }
1983                };
1984                dispatch_rows(pool, inter, &run);
1985                true
1986            }
1987            _ => false,
1988        }
1989    }
1990}
1991
1992/// Batched q8 kernel: same math as qmatvec, the row makes a single
1993/// pass from memory for the whole batch.
1994/// Accelerate CBLAS — the Apple AMX matrix units, the same engine
1995/// llama.cpp's `-ngl 0` prefill rides via ggml-blas.
1996#[cfg(target_os = "macos")]
1997mod accel_blas {
1998    #[link(name = "Accelerate", kind = "framework")]
1999    unsafe extern "C" {
2000        pub fn cblas_sgemm(
2001            order: i32,
2002            trans_a: i32,
2003            trans_b: i32,
2004            m: i32,
2005            n: i32,
2006            k: i32,
2007            alpha: f32,
2008            a: *const f32,
2009            lda: i32,
2010            b: *const f32,
2011            ldb: i32,
2012            beta: f32,
2013            c: *mut f32,
2014            ldc: i32,
2015        );
2016    }
2017}
2018
2019#[cfg(target_os = "macos")]
2020pub(crate) fn accel_gemm_enabled() -> bool {
2021    static ON: std::sync::OnceLock<bool> = std::sync::OnceLock::new();
2022    *ON.get_or_init(|| std::env::var("CMF_ACCEL").map(|v| v != "0").unwrap_or(true))
2023}
2024
2025/// Off macOS the "accel" GEMM is the portable NEON micro-kernel below —
2026/// same entry point, so the batched-attention path opens on mobile.
2027#[cfg(all(target_arch = "aarch64", not(target_os = "macos")))]
2028pub(crate) fn accel_gemm_enabled() -> bool {
2029    true
2030}
2031
2032/// Portable NEON f32 GEMM (row-major, optional Bᵀ): a 4×8 fmla
2033/// micro-kernel with A broadcast against B panels — the mobile stand-in
2034/// for Accelerate in the batched causal attention (QKᵀ and P·V). Not a
2035/// BLAS: shapes here are the attention panels (m ≤ heads·chunk,
2036/// k = head_dim or context), and the goal is removing the per-position
2037/// quadratic wall, not peak GEMM.
2038#[cfg(target_arch = "aarch64")]
2039#[allow(clippy::too_many_arguments)]
2040pub(crate) fn neon_gemm_rm(
2041    m: usize,
2042    n: usize,
2043    k: usize,
2044    alpha: f32,
2045    a: &[f32],
2046    lda: usize,
2047    b_mat: &[f32],
2048    ldb: usize,
2049    b_rows_are_n: bool,
2050    c: &mut [f32],
2051    ldc: usize,
2052) {
2053    debug_assert!(a.len() >= (m - 1) * lda + k);
2054    debug_assert!(c.len() >= (m - 1) * ldc + n);
2055    // SAFETY: bounds asserted above; NEON is baseline on aarch64.
2056    unsafe {
2057        use core::arch::aarch64::*;
2058        let mut i = 0usize;
2059        while i < m {
2060            let mi = (m - i).min(4);
2061            let mut j = 0usize;
2062            while j < n {
2063                let nj = (n - j).min(8);
2064                if mi == 4 && nj == 8 {
2065                    let (mut c0a, mut c0b) = (vdupq_n_f32(0.0), vdupq_n_f32(0.0));
2066                    let (mut c1a, mut c1b) = (vdupq_n_f32(0.0), vdupq_n_f32(0.0));
2067                    let (mut c2a, mut c2b) = (vdupq_n_f32(0.0), vdupq_n_f32(0.0));
2068                    let (mut c3a, mut c3b) = (vdupq_n_f32(0.0), vdupq_n_f32(0.0));
2069                    for p in 0..k {
2070                        let (b0, b1) = if b_rows_are_n {
2071                            // B is [n, k]: column p of Bᵀ = element p of
2072                            // eight consecutive B rows — gathered.
2073                            let base = b_mat.as_ptr().add(j * ldb + p);
2074                            let g = |o: usize| *base.add(o * ldb);
2075                            ([g(0), g(1), g(2), g(3)], [g(4), g(5), g(6), g(7)])
2076                        } else {
2077                            let base = b_mat.as_ptr().add(p * ldb + j);
2078                            (
2079                                [*base, *base.add(1), *base.add(2), *base.add(3)],
2080                                [*base.add(4), *base.add(5), *base.add(6), *base.add(7)],
2081                            )
2082                        };
2083                        let bv0 = vld1q_f32(b0.as_ptr());
2084                        let bv1 = vld1q_f32(b1.as_ptr());
2085                        let a0 = vdupq_n_f32(*a.as_ptr().add(i * lda + p));
2086                        let a1 = vdupq_n_f32(*a.as_ptr().add((i + 1) * lda + p));
2087                        let a2 = vdupq_n_f32(*a.as_ptr().add((i + 2) * lda + p));
2088                        let a3 = vdupq_n_f32(*a.as_ptr().add((i + 3) * lda + p));
2089                        c0a = vfmaq_f32(c0a, a0, bv0);
2090                        c0b = vfmaq_f32(c0b, a0, bv1);
2091                        c1a = vfmaq_f32(c1a, a1, bv0);
2092                        c1b = vfmaq_f32(c1b, a1, bv1);
2093                        c2a = vfmaq_f32(c2a, a2, bv0);
2094                        c2b = vfmaq_f32(c2b, a2, bv1);
2095                        c3a = vfmaq_f32(c3a, a3, bv0);
2096                        c3b = vfmaq_f32(c3b, a3, bv1);
2097                    }
2098                    let al = vdupq_n_f32(alpha);
2099                    for (r, (ca, cb)) in [(c0a, c0b), (c1a, c1b), (c2a, c2b), (c3a, c3b)]
2100                        .iter()
2101                        .enumerate()
2102                    {
2103                        let dst = c.as_mut_ptr().add((i + r) * ldc + j);
2104                        vst1q_f32(dst, vmulq_f32(*ca, al));
2105                        vst1q_f32(dst.add(4), vmulq_f32(*cb, al));
2106                    }
2107                } else {
2108                    for r in 0..mi {
2109                        for q in 0..nj {
2110                            let mut acc = 0f32;
2111                            for p in 0..k {
2112                                let bv = if b_rows_are_n {
2113                                    b_mat[(j + q) * ldb + p]
2114                                } else {
2115                                    b_mat[p * ldb + j + q]
2116                                };
2117                                acc += a[(i + r) * lda + p] * bv;
2118                            }
2119                            c[(i + r) * ldc + j + q] = acc * alpha;
2120                        }
2121                    }
2122                }
2123                j += nj;
2124            }
2125            i += mi;
2126        }
2127    }
2128}
2129
2130/// Off-macOS aarch64: the batched attention rides the NEON micro-GEMM.
2131#[cfg(all(target_arch = "aarch64", not(target_os = "macos")))]
2132#[allow(clippy::too_many_arguments)]
2133pub(crate) fn sgemm_rm(
2134    m: usize,
2135    n: usize,
2136    k: usize,
2137    alpha: f32,
2138    a: &[f32],
2139    lda: usize,
2140    b_mat: &[f32],
2141    ldb: usize,
2142    b_rows_are_n: bool,
2143    c: &mut [f32],
2144    ldc: usize,
2145) {
2146    neon_gemm_rm(m, n, k, alpha, a, lda, b_mat, ldb, b_rows_are_n, c, ldc);
2147}
2148
2149/// Row-major f32 GEMM on Accelerate: C[m,n] = alpha·A[m,k] × B(ᵀ).
2150/// `b_rows_are_n` = true multiplies by Bᵀ where B is stored [n, k].
2151#[cfg(target_os = "macos")]
2152#[allow(clippy::too_many_arguments)]
2153pub(crate) fn sgemm_rm(
2154    m: usize,
2155    n: usize,
2156    k: usize,
2157    alpha: f32,
2158    a: &[f32],
2159    lda: usize,
2160    b_mat: &[f32],
2161    ldb: usize,
2162    b_rows_are_n: bool,
2163    c: &mut [f32],
2164    ldc: usize,
2165) {
2166    debug_assert!(a.len() >= (m - 1) * lda + k);
2167    debug_assert!(c.len() >= (m - 1) * ldc + n);
2168    // Test hook: route the attention GEMMs through the portable NEON
2169    // micro-kernel ON APPLE SILICON — how the mobile batched attend is
2170    // measured without a phone in the loop. (Intel macOS has no NEON —
2171    // the hook is a no-op there, Accelerate continues below.)
2172    #[cfg(target_arch = "aarch64")]
2173    if std::env::var("CMF_FORCE_NEON_GEMM")
2174        .map(|v| v == "1")
2175        .unwrap_or(false)
2176    {
2177        return neon_gemm_rm(m, n, k, alpha, a, lda, b_mat, ldb, b_rows_are_n, c, ldc);
2178    }
2179    unsafe {
2180        accel_blas::cblas_sgemm(
2181            101, // RowMajor
2182            111, // NoTrans A
2183            if b_rows_are_n { 112 } else { 111 },
2184            m as i32,
2185            n as i32,
2186            k as i32,
2187            alpha,
2188            a.as_ptr(),
2189            lda as i32,
2190            b_mat.as_ptr(),
2191            ldb as i32,
2192            0.0,
2193            c.as_mut_ptr(),
2194            ldc as i32,
2195        );
2196    }
2197}
2198
2199/// Prefill GEMM through Accelerate (macOS): dequantize q8 rows into
2200/// f32 tiles (scale folded in, pool-parallel) and multiply each tile
2201/// on the AMX with one row-major sgemm. Tiles live in cache, weights
2202/// stream once. Numerics are f32-GEMM (not the int8 dot): prefill
2203/// logits shift within f32 rounding — tolerance-class, like every
2204/// reduction-order change; decode (M=1) never takes this path.
2205#[cfg(target_os = "macos")]
2206fn qmatmat_accel(
2207    q: &[u8],
2208    row_scale: &[f32],
2209    pre: &[std::borrow::Cow<'_, [f32]>],
2210    rows: usize,
2211    cols: usize,
2212    out: &mut [f32],
2213    pool: Option<&Pool>,
2214) {
2215    // NOTE: double-buffering the dequant against the sgemm (a scoped
2216    // thread driving the pool on tile k+1 while the caller multiplies
2217    // tile k) was tried and LOST ~6%: Accelerate's sgemm is itself
2218    // multithreaded, and the dequant workers just steal its cores.
2219    const TR: usize = 2048;
2220    let b = pre.len();
2221    thread_local! {
2222        static XPANEL: std::cell::RefCell<Vec<f32>> = const { std::cell::RefCell::new(Vec::new()) };
2223        static WTILE: std::cell::RefCell<Vec<f32>> = const { std::cell::RefCell::new(Vec::new()) };
2224    }
2225    XPANEL.with(|xp| {
2226        WTILE.with(|wt| {
2227            let mut xpanel = xp.borrow_mut();
2228            xpanel.clear();
2229            for x in pre {
2230                xpanel.extend_from_slice(x);
2231            }
2232            let mut wtile = wt.borrow_mut();
2233            wtile.resize(TR * cols, 0.0);
2234            let mut r0 = 0usize;
2235            while r0 < rows {
2236                let tr = TR.min(rows - r0);
2237                // Dequant the tile (scale folded) — pool-parallel.
2238                let wt_addr = SendMut(wtile.as_mut_ptr());
2239                let run = |start: usize, end: usize| {
2240                    for r in start..end {
2241                        let row = &q[(r0 + r) * cols..(r0 + r + 1) * cols];
2242                        let s = row_scale[r0 + r];
2243                        // SAFETY: workers cover disjoint r ranges.
2244                        let dst =
2245                            unsafe { std::slice::from_raw_parts_mut(wt_addr.at(r * cols), cols) };
2246                        for (d, &v) in dst.iter_mut().zip(row) {
2247                            *d = (v as i8) as f32 * s;
2248                        }
2249                    }
2250                };
2251                dispatch_rows(pool, tr, &run);
2252                // C[b, tr] (at column r0 of out[b, rows]) = X · Wtileᵀ
2253                unsafe {
2254                    accel_blas::cblas_sgemm(
2255                        101, // RowMajor
2256                        111, // NoTrans A
2257                        112, // Trans B
2258                        b as i32,
2259                        tr as i32,
2260                        cols as i32,
2261                        1.0,
2262                        xpanel.as_ptr(),
2263                        cols as i32,
2264                        wtile.as_ptr(),
2265                        cols as i32,
2266                        0.0,
2267                        out.as_mut_ptr().add(r0),
2268                        rows as i32,
2269                    );
2270                }
2271                r0 += tr;
2272            }
2273        })
2274    });
2275}
2276
2277fn qmatmat(
2278    q: &[u8],
2279    row_scale: &[f32],
2280    pre: &[std::borrow::Cow<'_, [f32]>],
2281    rows: usize,
2282    cols: usize,
2283    out: &mut [f32],
2284    pool: Option<&Pool>,
2285) {
2286    let b = pre.len();
2287    debug_assert_eq!(out.len(), b * rows);
2288    // Big prefill batches ride the AMX (roadmap PR3): the row×batch
2289    // SDOT loop below peaks near the CPU's dot throughput, an order
2290    // below the matrix units. Small tensors and tiny test models stay
2291    // on the exact integer path.
2292    #[cfg(target_os = "macos")]
2293    if b >= 8 && rows * cols >= 500_000 && accel_gemm_enabled() {
2294        qmatmat_accel(q, row_scale, pre, rows, cols, out, pool);
2295        return;
2296    }
2297    #[cfg(target_arch = "aarch64")]
2298    if sdot_enabled() {
2299        let acts: Vec<SplitAct> = pre.iter().map(|x| split_act(x)).collect();
2300        let out_addr = SendMut(out.as_mut_ptr());
2301        // Blocked 2×4 (mobile prefill: no AMX to fall back on — this
2302        // path IS the ARM prefill GEMM off Apple silicon).
2303        let blocked_ok = std::env::var("CMF_X86_BLOCKED")
2304            .map(|v| v != "0")
2305            .unwrap_or(true);
2306        let use_i8mm = i8mm_enabled();
2307        if blocked_ok {
2308            let run = |start: usize, end: usize| {
2309                let mut o = start;
2310                while o < end {
2311                    if o + 2 <= end {
2312                        let r0 = &q[o * cols..(o + 1) * cols];
2313                        let r1 = &q[(o + 1) * cols..(o + 2) * cols];
2314                        let mut bi = 0usize;
2315                        while bi + 4 <= acts.len() {
2316                            let xs = [
2317                                acts[bi].xq.as_slice(),
2318                                acts[bi + 1].xq.as_slice(),
2319                                acts[bi + 2].xq.as_slice(),
2320                                acts[bi + 3].xq.as_slice(),
2321                            ];
2322                            let d = if use_i8mm {
2323                                unsafe { dot_i8_smmla_2x4(r0, r1, xs) }
2324                            } else {
2325                                unsafe { dot_i8_sdot_2x4(r0, r1, xs) }
2326                            };
2327                            for (r, row) in [r0, r1].into_iter().enumerate() {
2328                                for k in 0..4 {
2329                                    let act = &acts[bi + k];
2330                                    let mut v = d[r][k] as f32 * act.sx;
2331                                    for &(j, xv) in &act.outliers {
2332                                        v += (row[j] as i8) as f32 * xv;
2333                                    }
2334                                    unsafe {
2335                                        *out_addr.at((bi + k) * rows + o + r) = v * row_scale[o + r]
2336                                    };
2337                                }
2338                            }
2339                            bi += 4;
2340                        }
2341                        while bi < acts.len() {
2342                            for (r, row) in [r0, r1].into_iter().enumerate() {
2343                                let v = row_dot_sdot(row, &acts[bi]) * row_scale[o + r];
2344                                unsafe { *out_addr.at(bi * rows + o + r) = v };
2345                            }
2346                            bi += 1;
2347                        }
2348                        o += 2;
2349                    } else {
2350                        let row = &q[o * cols..(o + 1) * cols];
2351                        for (bi, act) in acts.iter().enumerate() {
2352                            let v = row_dot_sdot(row, act) * row_scale[o];
2353                            unsafe { *out_addr.at(bi * rows + o) = v };
2354                        }
2355                        o += 1;
2356                    }
2357                }
2358            };
2359            dispatch_rows(pool, rows, &run);
2360            return;
2361        }
2362        let run = |start: usize, end: usize| {
2363            for o in start..end {
2364                let row = &q[o * cols..(o + 1) * cols];
2365                for (bi, act) in acts.iter().enumerate() {
2366                    let v = row_dot_sdot(row, act) * row_scale[o];
2367                    unsafe { *out_addr.at(bi * rows + o) = v };
2368                }
2369            }
2370        };
2371        dispatch_rows(pool, rows, &run);
2372        return;
2373    }
2374    // x86 A8W8 batch. Non-VNNI parts take the BLOCKED 2×4 kernel
2375    // (roadmap P0: two weight rows' abs() stay in registers across four
2376    // activation streams); VNNI machines keep the per-row bias-trick
2377    // dot, which is already throughput-bound there.
2378    #[cfg(target_arch = "x86_64")]
2379    if avx2_a8w8_enabled() {
2380        let acts: Vec<SplitAct> = pre.iter().map(|x| split_act(x)).collect();
2381        let out_addr = SendMut(out.as_mut_ptr());
2382        // CMF_X86_BLOCKED=0 forces the per-row path (paired in-process
2383        // A/B on noisy shared-vCPU hosts).
2384        let blocked_ok = std::env::var("CMF_X86_BLOCKED")
2385            .map(|v| v != "0")
2386            .unwrap_or(true);
2387        if !avx512vnni_enabled() && blocked_ok {
2388            let run = |start: usize, end: usize| {
2389                let mut o = start;
2390                while o < end {
2391                    if o + 2 <= end {
2392                        let r0 = &q[o * cols..(o + 1) * cols];
2393                        let r1 = &q[(o + 1) * cols..(o + 2) * cols];
2394                        let mut bi = 0usize;
2395                        while bi + 4 <= acts.len() {
2396                            let xs = [
2397                                acts[bi].xq.as_slice(),
2398                                acts[bi + 1].xq.as_slice(),
2399                                acts[bi + 2].xq.as_slice(),
2400                                acts[bi + 3].xq.as_slice(),
2401                            ];
2402                            let d = unsafe { dot_i8_i8_avx2_2x4(r0, r1, xs) };
2403                            for (r, row) in [r0, r1].into_iter().enumerate() {
2404                                for k in 0..4 {
2405                                    let act = &acts[bi + k];
2406                                    let mut v = d[r][k] as f32 * act.sx;
2407                                    for &(j, xv) in &act.outliers {
2408                                        v += (row[j] as i8) as f32 * xv;
2409                                    }
2410                                    unsafe {
2411                                        *out_addr.at((bi + k) * rows + o + r) = v * row_scale[o + r]
2412                                    };
2413                                }
2414                            }
2415                            bi += 4;
2416                        }
2417                        while bi < acts.len() {
2418                            for (r, row) in [r0, r1].into_iter().enumerate() {
2419                                let v = row_dot_avx2(row, &acts[bi]) * row_scale[o + r];
2420                                unsafe { *out_addr.at(bi * rows + o + r) = v };
2421                            }
2422                            bi += 1;
2423                        }
2424                        o += 2;
2425                    } else {
2426                        let row = &q[o * cols..(o + 1) * cols];
2427                        for (bi, act) in acts.iter().enumerate() {
2428                            let v = row_dot_avx2(row, act) * row_scale[o];
2429                            unsafe { *out_addr.at(bi * rows + o) = v };
2430                        }
2431                        o += 1;
2432                    }
2433                }
2434            };
2435            dispatch_rows(pool, rows, &run);
2436            return;
2437        }
2438        let run = |start: usize, end: usize| {
2439            for o in start..end {
2440                let row = &q[o * cols..(o + 1) * cols];
2441                for (bi, act) in acts.iter().enumerate() {
2442                    let v = row_dot_avx2(row, act) * row_scale[o];
2443                    unsafe { *out_addr.at(bi * rows + o) = v };
2444                }
2445            }
2446        };
2447        dispatch_rows(pool, rows, &run);
2448        return;
2449    }
2450    let out_addr = SendMut(out.as_mut_ptr());
2451    let run = |start: usize, end: usize| {
2452        for o in start..end {
2453            let row = &q[o * cols..(o + 1) * cols];
2454            for (bi, x) in pre.iter().enumerate() {
2455                let mut acc = 0f32;
2456                for j in 0..cols {
2457                    acc += (row[j] as i8) as f32 * x[j];
2458                }
2459                unsafe { *out_addr.at(bi * rows + o) = acc * row_scale[o] };
2460            }
2461        }
2462    };
2463    dispatch_rows(pool, rows, &run);
2464}
2465
2466/// Split rows across pool workers (shared qmatvec pattern). Self-balancing
2467/// — see `Pool::run_rows` for why a static 1/n split is wrong here.
2468fn dispatch_rows(pool: Option<&Pool>, rows: usize, run: &(dyn Fn(usize, usize) + Sync)) {
2469    match pool {
2470        Some(pool) if rows >= 256 => pool.run_rows(rows, run),
2471        _ => run(0, rows),
2472    }
2473}
2474
2475/// Split a q4_block blob into (packed nibbles, f16 group scales).
2476fn q4_split(bytes: &[u8], rows: usize, cols: usize) -> (&[u8], &[u8]) {
2477    let groups = rows * cols / GROUP_SIZE;
2478    bytes.split_at(groups * 16)
2479}
2480
2481/// SIMD unpack for the dominant vbit width B=4 (94% of rows on the
2482/// log2-shape calibration): 16 packed bytes -> 32 centered i8 values.
2483/// vbit packs MSB-first, so the HIGH nibble is the even element
2484/// (opposite of q4_block's lo-first interleave). Centering is u-7.
2485#[inline]
2486fn vbit_fill4(data: &[u8], buf: &mut [u8]) {
2487    #[cfg(target_arch = "aarch64")]
2488    unsafe {
2489        return vbit_fill4_neon(data, buf);
2490    }
2491    #[cfg(target_arch = "x86_64")]
2492    if avx2_enabled() {
2493        return unsafe { vbit_fill4_avx2(data, buf) };
2494    }
2495    #[allow(unreachable_code)]
2496    for (blk, chunk) in buf.chunks_exact_mut(8).enumerate() {
2497        let u = unpack8::<4>(&data[blk * 4..]);
2498        for k in 0..8 {
2499            chunk[k] = (u[k] - 7) as i8 as u8;
2500        }
2501    }
2502}
2503
2504#[cfg(target_arch = "aarch64")]
2505#[target_feature(enable = "neon")]
2506unsafe fn vbit_fill4_neon(data: &[u8], buf: &mut [u8]) {
2507    // SAFETY: buf.len() is a multiple of GROUP_SIZE=32; data holds
2508    // buf.len()/2 packed bytes (validated at load).
2509    unsafe {
2510        use core::arch::aarch64::*;
2511        let n = buf.len();
2512        let mask = vdupq_n_u8(0x0F);
2513        let seven = vdupq_n_s8(7);
2514        let mut g = 0usize;
2515        while g * 32 + 32 <= n {
2516            let b = vld1q_u8(data.as_ptr().add(g * 16));
2517            let hi = vshrq_n_u8::<4>(b);
2518            let lo = vandq_u8(b, mask);
2519            let z0 = vsubq_s8(vreinterpretq_s8_u8(vzip1q_u8(hi, lo)), seven);
2520            let z1 = vsubq_s8(vreinterpretq_s8_u8(vzip2q_u8(hi, lo)), seven);
2521            vst1q_u8(buf.as_mut_ptr().add(g * 32), vreinterpretq_u8_s8(z0));
2522            vst1q_u8(buf.as_mut_ptr().add(g * 32 + 16), vreinterpretq_u8_s8(z1));
2523            g += 1;
2524        }
2525    }
2526}
2527
2528#[cfg(target_arch = "x86_64")]
2529#[target_feature(enable = "avx2")]
2530unsafe fn vbit_fill4_avx2(data: &[u8], buf: &mut [u8]) {
2531    // SAFETY: see vbit_fill4_neon.
2532    unsafe {
2533        use core::arch::x86_64::*;
2534        let n = buf.len();
2535        let mask = _mm_set1_epi8(0x0F);
2536        let seven = _mm256_set1_epi8(7);
2537        let mut g = 0usize;
2538        while g * 32 + 32 <= n {
2539            let b = _mm_loadu_si128(data.as_ptr().add(g * 16) as *const __m128i);
2540            let hi = _mm_and_si128(_mm_srli_epi16::<4>(b), mask);
2541            let lo = _mm_and_si128(b, mask);
2542            let z = _mm256_sub_epi8(
2543                _mm256_set_m128i(_mm_unpackhi_epi8(hi, lo), _mm_unpacklo_epi8(hi, lo)),
2544                seven,
2545            );
2546            _mm256_storeu_si256(buf.as_mut_ptr().add(g * 32) as *mut __m256i, z);
2547            g += 1;
2548        }
2549    }
2550}
2551
2552/// Unpack 8 MSB-first B-bit values from exactly B bytes (fixed shifts —
2553/// no serial bit-buffer, auto-vectorizable). Every 32-value group starts
2554/// byte-aligned (32·B/8 is integral for B∈3..8), so groups decompose
2555/// into 4 such blocks.
2556#[inline(always)]
2557fn unpack8<const B: usize>(data: &[u8]) -> [i32; 8] {
2558    let mut acc = 0u64;
2559    for i in 0..B {
2560        acc = (acc << 8) | data[i] as u64;
2561    }
2562    let mask = (1u64 << B) - 1;
2563    let mut out = [0i32; 8];
2564    for (k, o) in out.iter_mut().enumerate() {
2565        *o = ((acc >> ((7 - k) * B)) & mask) as i32;
2566    }
2567    out
2568}
2569
2570/// Fused vbit matvec straight from the mapped bytes (spec §3, P13
2571/// FIG.3): [u8 bits: rows][f16 scales: rows·cols/32][bit-packed rows,
2572/// MSB-first, byte-padded]. Row data offsets are precomputed at load
2573/// (`vbit_row_offsets`) — the per-call prefix scan was O(rows) pure
2574/// overhead on every matvec.
2575#[allow(clippy::too_many_arguments)]
2576fn vbitmatvec(
2577    bytes: &[u8],
2578    offsets: &[usize],
2579    x: &[f32],
2580    rows: usize,
2581    cols: usize,
2582    out: &mut [f32],
2583    pool: Option<&Pool>,
2584) {
2585    debug_assert_eq!(out.len(), rows);
2586    debug_assert_eq!(offsets.len(), rows + 1);
2587
2588    // SDOT path: unpack the row to centered i8 once, then per-group
2589    // int8 dot against the quantized activations — same A8W8 contract
2590    // as q8 (bounded noise; CMF_SDOT=0 keeps the exact scalar path).
2591    if a8w8_enabled() {
2592        let act = split_act(x);
2593        let out_addr = SendMut(out.as_mut_ptr());
2594        let run = move |start: usize, end: usize| {
2595            vbit_range_a8w8(bytes, offsets, x, &act, rows, cols, out_addr, start, end)
2596        };
2597        dispatch_rows(pool, rows, &run);
2598        return;
2599    }
2600
2601    let out_addr = SendMut(out.as_mut_ptr());
2602    let run = move |start: usize, end: usize| {
2603        vbit_range_f32(bytes, offsets, x, rows, cols, out_addr, start, end)
2604    };
2605    dispatch_rows(pool, rows, &run);
2606}
2607
2608/// One vbit row range via the A8W8 int8 path — kernel body of
2609/// `vbitmatvec`, extracted so multi-matrix jobs can drive it for
2610/// several tensors in one dispatch (b=8 rows go exact f32).
2611#[allow(clippy::too_many_arguments)]
2612fn vbit_range_a8w8(
2613    bytes: &[u8],
2614    offsets: &[usize],
2615    x: &[f32],
2616    act: &SplitAct,
2617    rows: usize,
2618    cols: usize,
2619    out: SendMut,
2620    start: usize,
2621    end: usize,
2622) {
2623    let ng = cols / GROUP_SIZE;
2624    let bits = &bytes[..rows];
2625    let sc_off = rows;
2626    let row_dot = |r: usize| -> f32 {
2627        let b = bits[r] as usize;
2628        let l = (1i32 << (b - 1)) - 1;
2629        let mask = (1u64 << b) - 1;
2630        let data = &bytes[offsets[r]..offsets[r + 1]];
2631        if b == 8 {
2632            // u−L reaches 128 → does not fit i8; exact f32 path.
2633            let (mut acc, mut nbits, mut idx) = (0u64, 0usize, 0usize);
2634            let mut dot = 0f32;
2635            for g in 0..ng {
2636                let so = (r * ng + g) * 2;
2637                let sgf = f16_to_f32(u16::from_le_bytes([
2638                    bytes[sc_off + so],
2639                    bytes[sc_off + so + 1],
2640                ]));
2641                let xg = &x[g * GROUP_SIZE..(g + 1) * GROUP_SIZE];
2642                let mut gd = 0f32;
2643                for &xv in xg.iter() {
2644                    if nbits < 8 {
2645                        acc = (acc << 8) | data[idx] as u64;
2646                        idx += 1;
2647                        nbits += 8;
2648                    }
2649                    let u = ((acc >> (nbits - 8)) & 0xFF) as i32;
2650                    nbits -= 8;
2651                    gd += (u - l) as f32 * xv;
2652                }
2653                dot += gd * sgf;
2654            }
2655            return dot;
2656        }
2657        // Per-worker scratch: this closure runs for every row of the
2658        // tensor (lm_head ≈ 150k rows/token) — a heap allocation per
2659        // row was measurable pure overhead.
2660        thread_local! {
2661            static VBIT_SCRATCH: std::cell::RefCell<Vec<u8>> =
2662                const { std::cell::RefCell::new(Vec::new()) };
2663        }
2664        #[inline(always)]
2665        fn fill<const B: usize>(data: &[u8], l: i32, buf: &mut [u8]) {
2666            for (blk, chunk) in buf.chunks_exact_mut(8).enumerate() {
2667                let u = unpack8::<B>(&data[blk * B..]);
2668                for k in 0..8 {
2669                    chunk[k] = (u[k] - l) as i8 as u8;
2670                }
2671            }
2672        }
2673        let _ = mask;
2674        VBIT_SCRATCH.with(|scratch| {
2675            let mut buf = scratch.borrow_mut();
2676            buf.resize(cols, 0);
2677            match b {
2678                3 => fill::<3>(data, l, &mut buf),
2679                4 => vbit_fill4(data, &mut buf),
2680                5 => fill::<5>(data, l, &mut buf),
2681                6 => fill::<6>(data, l, &mut buf),
2682                _ => unreachable!(),
2683            }
2684            let mut dot = 0f32;
2685            for g in 0..ng {
2686                let so = (r * ng + g) * 2;
2687                let s = f16_to_f32(u16::from_le_bytes([
2688                    bytes[sc_off + so],
2689                    bytes[sc_off + so + 1],
2690                ]));
2691                let d = dot_i8_i8(
2692                    &buf[g * GROUP_SIZE..(g + 1) * GROUP_SIZE],
2693                    &act.xq[g * GROUP_SIZE..(g + 1) * GROUP_SIZE],
2694                ) as f32
2695                    * act.sx;
2696                dot += d * s;
2697            }
2698            for &(j, xv) in &act.outliers {
2699                let so = (r * ng + j / GROUP_SIZE) * 2;
2700                let s = f16_to_f32(u16::from_le_bytes([
2701                    bytes[sc_off + so],
2702                    bytes[sc_off + so + 1],
2703                ]));
2704                // xq is zeroed at outlier slots — add the exact term.
2705                dot += (buf[j] as i8) as f32 * s * xv;
2706            }
2707            dot
2708        })
2709    };
2710    for r in start..end {
2711        // SAFETY: disjoint row ranges per worker.
2712        unsafe { *out.at(r) = row_dot(r) };
2713    }
2714}
2715
2716/// Exact scalar vbit row range (same extraction, non-SDOT path).
2717#[allow(clippy::too_many_arguments)]
2718fn vbit_range_f32(
2719    bytes: &[u8],
2720    offsets: &[usize],
2721    x: &[f32],
2722    rows: usize,
2723    cols: usize,
2724    out: SendMut,
2725    start: usize,
2726    end: usize,
2727) {
2728    let ng = cols / GROUP_SIZE;
2729    let bits = &bytes[..rows];
2730    let sc_off = rows;
2731    // Per-bit-width specialized inner loops: the compiler unrolls the
2732    // constant shifts (the generic bit-buffer loop was branch-bound —
2733    // 5.6 vs 13.2 tok/s q4 on the 0.8B).
2734    #[inline(always)]
2735    fn dot_row<const B: usize>(
2736        data: &[u8],
2737        bytes: &[u8],
2738        sc_off: usize,
2739        r: usize,
2740        ng: usize,
2741        x: &[f32],
2742    ) -> f32 {
2743        let l = ((1i32 << (B - 1)) - 1) as f32;
2744        let gbytes = GROUP_SIZE * B / 8;
2745        let mut dot = 0f32;
2746        for g in 0..ng {
2747            let so = (r * ng + g) * 2;
2748            let s = f16_to_f32(u16::from_le_bytes([
2749                bytes[sc_off + so],
2750                bytes[sc_off + so + 1],
2751            ]));
2752            let xg = &x[g * GROUP_SIZE..(g + 1) * GROUP_SIZE];
2753            let gd0 = &data[g * gbytes..(g + 1) * gbytes];
2754            let mut gd = 0f32;
2755            for blk in 0..GROUP_SIZE / 8 {
2756                let u = unpack8::<B>(&gd0[blk * B..]);
2757                let xb = &xg[blk * 8..blk * 8 + 8];
2758                for k in 0..8 {
2759                    gd += (u[k] as f32 - l) * xb[k];
2760                }
2761            }
2762            dot += gd * s;
2763        }
2764        dot
2765    }
2766    for r in start..end {
2767        let data = &bytes[offsets[r]..offsets[r + 1]];
2768        let v = match bits[r] {
2769            3 => dot_row::<3>(data, bytes, sc_off, r, ng, x),
2770            4 => dot_row::<4>(data, bytes, sc_off, r, ng, x),
2771            5 => dot_row::<5>(data, bytes, sc_off, r, ng, x),
2772            6 => dot_row::<6>(data, bytes, sc_off, r, ng, x),
2773            8 => dot_row::<8>(data, bytes, sc_off, r, ng, x),
2774            b => unreachable!("vbit bit-width {b} (validated at load)"),
2775        };
2776        // SAFETY: disjoint row ranges per worker.
2777        unsafe { *out.at(r) = v };
2778    }
2779}
2780
2781/// Fused two-input vbit matvec: each row is unpacked from the mmap ONCE
2782/// and dotted against BOTH activations (MTP verify / pair prefill used
2783/// to run two full matvecs — double weight traffic and double unpack).
2784/// Per-input math is identical to `vbitmatvec` → same accuracy contract.
2785#[allow(clippy::too_many_arguments)]
2786fn vbitmatvec2(
2787    bytes: &[u8],
2788    offsets: &[usize],
2789    x1: &[f32],
2790    x2: &[f32],
2791    rows: usize,
2792    cols: usize,
2793    o1: &mut [f32],
2794    o2: &mut [f32],
2795    pool: Option<&Pool>,
2796) {
2797    debug_assert_eq!(o1.len(), rows);
2798    debug_assert_eq!(o2.len(), rows);
2799
2800    if a8w8_enabled() {
2801        let a1 = split_act(x1);
2802        let a2 = split_act(x2);
2803        let p1 = SendMut(o1.as_mut_ptr());
2804        let p2 = SendMut(o2.as_mut_ptr());
2805        let run = move |start: usize, end: usize| {
2806            vbit_range2_a8w8(
2807                bytes, offsets, x1, x2, &a1, &a2, rows, cols, p1, p2, start, end,
2808            )
2809        };
2810        dispatch_rows(pool, rows, &run);
2811        return;
2812    }
2813
2814    let p1 = SendMut(o1.as_mut_ptr());
2815    let p2 = SendMut(o2.as_mut_ptr());
2816    let run = move |start: usize, end: usize| {
2817        vbit_range2_f32(bytes, offsets, x1, x2, rows, cols, p1, p2, start, end)
2818    };
2819    dispatch_rows(pool, rows, &run);
2820}
2821
2822/// Two-input vbit row range via the A8W8 int8 path — kernel body of
2823/// `vbitmatvec2`, extracted for pair multi-matrix jobs (b=8 rows go
2824/// exact f32 for both lanes, bits streamed once).
2825#[allow(clippy::too_many_arguments)]
2826fn vbit_range2_a8w8(
2827    bytes: &[u8],
2828    offsets: &[usize],
2829    x1: &[f32],
2830    x2: &[f32],
2831    a1: &SplitAct,
2832    a2: &SplitAct,
2833    rows: usize,
2834    cols: usize,
2835    p1: SendMut,
2836    p2: SendMut,
2837    start: usize,
2838    end: usize,
2839) {
2840    let ng = cols / GROUP_SIZE;
2841    let bits = &bytes[..rows];
2842    let sc_off = rows;
2843    let row_dots = |r: usize| -> (f32, f32) {
2844        let b = bits[r] as usize;
2845        let l = (1i32 << (b - 1)) - 1;
2846        let data = &bytes[offsets[r]..offsets[r + 1]];
2847        if b == 8 {
2848            // u−L reaches 128 → does not fit i8; exact f32 path,
2849            // bits still streamed once for both lanes.
2850            let (mut acc, mut nbits, mut idx) = (0u64, 0usize, 0usize);
2851            let (mut d1, mut d2) = (0f32, 0f32);
2852            for g in 0..ng {
2853                let so = (r * ng + g) * 2;
2854                let sgf = f16_to_f32(u16::from_le_bytes([
2855                    bytes[sc_off + so],
2856                    bytes[sc_off + so + 1],
2857                ]));
2858                let (mut g1, mut g2) = (0f32, 0f32);
2859                for k in 0..GROUP_SIZE {
2860                    if nbits < 8 {
2861                        acc = (acc << 8) | data[idx] as u64;
2862                        idx += 1;
2863                        nbits += 8;
2864                    }
2865                    let u = ((acc >> (nbits - 8)) & 0xFF) as i32;
2866                    nbits -= 8;
2867                    let w = (u - l) as f32;
2868                    g1 += w * x1[g * GROUP_SIZE + k];
2869                    g2 += w * x2[g * GROUP_SIZE + k];
2870                }
2871                d1 += g1 * sgf;
2872                d2 += g2 * sgf;
2873            }
2874            return (d1, d2);
2875        }
2876        thread_local! {
2877            static VBIT_SCRATCH2: std::cell::RefCell<Vec<u8>> =
2878                const { std::cell::RefCell::new(Vec::new()) };
2879        }
2880        #[inline(always)]
2881        fn fill<const B: usize>(data: &[u8], l: i32, buf: &mut [u8]) {
2882            for (blk, chunk) in buf.chunks_exact_mut(8).enumerate() {
2883                let u = unpack8::<B>(&data[blk * B..]);
2884                for k in 0..8 {
2885                    chunk[k] = (u[k] - l) as i8 as u8;
2886                }
2887            }
2888        }
2889        VBIT_SCRATCH2.with(|scratch| {
2890            let mut buf = scratch.borrow_mut();
2891            buf.resize(cols, 0);
2892            match b {
2893                3 => fill::<3>(data, l, &mut buf),
2894                4 => vbit_fill4(data, &mut buf),
2895                5 => fill::<5>(data, l, &mut buf),
2896                6 => fill::<6>(data, l, &mut buf),
2897                _ => unreachable!(),
2898            }
2899            let (mut d1, mut d2) = (0f32, 0f32);
2900            for g in 0..ng {
2901                let so = (r * ng + g) * 2;
2902                let s = f16_to_f32(u16::from_le_bytes([
2903                    bytes[sc_off + so],
2904                    bytes[sc_off + so + 1],
2905                ]));
2906                let wg = &buf[g * GROUP_SIZE..(g + 1) * GROUP_SIZE];
2907                let v1 = dot_i8_i8(wg, &a1.xq[g * GROUP_SIZE..(g + 1) * GROUP_SIZE]) as f32 * a1.sx;
2908                let v2 = dot_i8_i8(wg, &a2.xq[g * GROUP_SIZE..(g + 1) * GROUP_SIZE]) as f32 * a2.sx;
2909                d1 += v1 * s;
2910                d2 += v2 * s;
2911            }
2912            for &(j, xv) in &a1.outliers {
2913                let so = (r * ng + j / GROUP_SIZE) * 2;
2914                let s = f16_to_f32(u16::from_le_bytes([
2915                    bytes[sc_off + so],
2916                    bytes[sc_off + so + 1],
2917                ]));
2918                d1 += (buf[j] as i8) as f32 * s * xv;
2919            }
2920            for &(j, xv) in &a2.outliers {
2921                let so = (r * ng + j / GROUP_SIZE) * 2;
2922                let s = f16_to_f32(u16::from_le_bytes([
2923                    bytes[sc_off + so],
2924                    bytes[sc_off + so + 1],
2925                ]));
2926                d2 += (buf[j] as i8) as f32 * s * xv;
2927            }
2928            (d1, d2)
2929        })
2930    };
2931    for r in start..end {
2932        let (v1, v2) = row_dots(r);
2933        // SAFETY: disjoint row ranges per worker.
2934        unsafe {
2935            *p1.at(r) = v1;
2936            *p2.at(r) = v2;
2937        }
2938    }
2939}
2940
2941/// Two-input exact scalar vbit row range (same extraction) —
2942/// per-bit-width specialized, two accumulators per row; per-lane
2943/// accumulation order matches `vbitmatvec` exactly.
2944#[allow(clippy::too_many_arguments)]
2945fn vbit_range2_f32(
2946    bytes: &[u8],
2947    offsets: &[usize],
2948    x1: &[f32],
2949    x2: &[f32],
2950    rows: usize,
2951    cols: usize,
2952    p1: SendMut,
2953    p2: SendMut,
2954    start: usize,
2955    end: usize,
2956) {
2957    let ng = cols / GROUP_SIZE;
2958    let bits = &bytes[..rows];
2959    let sc_off = rows;
2960    #[inline(always)]
2961    #[allow(clippy::too_many_arguments)]
2962    fn dot_row2<const B: usize>(
2963        data: &[u8],
2964        bytes: &[u8],
2965        sc_off: usize,
2966        r: usize,
2967        ng: usize,
2968        x1: &[f32],
2969        x2: &[f32],
2970    ) -> (f32, f32) {
2971        let l = ((1i32 << (B - 1)) - 1) as f32;
2972        let gbytes = GROUP_SIZE * B / 8;
2973        let (mut d1, mut d2) = (0f32, 0f32);
2974        for g in 0..ng {
2975            let so = (r * ng + g) * 2;
2976            let s = f16_to_f32(u16::from_le_bytes([
2977                bytes[sc_off + so],
2978                bytes[sc_off + so + 1],
2979            ]));
2980            let x1g = &x1[g * GROUP_SIZE..(g + 1) * GROUP_SIZE];
2981            let x2g = &x2[g * GROUP_SIZE..(g + 1) * GROUP_SIZE];
2982            let gd0 = &data[g * gbytes..(g + 1) * gbytes];
2983            let (mut g1, mut g2) = (0f32, 0f32);
2984            for blk in 0..GROUP_SIZE / 8 {
2985                let u = unpack8::<B>(&gd0[blk * B..]);
2986                for k in 0..8 {
2987                    let w = u[k] as f32 - l;
2988                    g1 += w * x1g[blk * 8 + k];
2989                    g2 += w * x2g[blk * 8 + k];
2990                }
2991            }
2992            d1 += g1 * s;
2993            d2 += g2 * s;
2994        }
2995        (d1, d2)
2996    }
2997    for r in start..end {
2998        let data = &bytes[offsets[r]..offsets[r + 1]];
2999        let (v1, v2) = match bits[r] {
3000            3 => dot_row2::<3>(data, bytes, sc_off, r, ng, x1, x2),
3001            4 => dot_row2::<4>(data, bytes, sc_off, r, ng, x1, x2),
3002            5 => dot_row2::<5>(data, bytes, sc_off, r, ng, x1, x2),
3003            6 => dot_row2::<6>(data, bytes, sc_off, r, ng, x1, x2),
3004            8 => dot_row2::<8>(data, bytes, sc_off, r, ng, x1, x2),
3005            b => unreachable!("vbit bit-width {b} (validated at load)"),
3006        };
3007        // SAFETY: disjoint row ranges per worker.
3008        unsafe {
3009            *p1.at(r) = v1;
3010            *p2.at(r) = v2;
3011        }
3012    }
3013}
3014
3015// ───────────────────── q4_tiled kernels (§4.3) ─────────────────────
3016
3017/// One q4_tiled row dot on the A8W8 int8 path: per 32-group the tile
3018/// is ONE sequential read — [f16 scale][16B nibbles] — versus the two
3019/// distant streams of the split layout. Values/order identical to the
3020/// split kernels.
3021#[inline]
3022#[allow(unreachable_code)]
3023fn dot_q4t_row_i8(bytes: &[u8], r: usize, gpr: usize, xq: &[i8]) -> f32 {
3024    #[cfg(target_arch = "aarch64")]
3025    unsafe {
3026        return dot_q4t_row_sdot(bytes, r, gpr, xq);
3027    }
3028    #[cfg(target_arch = "x86_64")]
3029    unsafe {
3030        if vnni_tiles_enabled() {
3031            return dot_q4t_row_vnni(bytes, r, gpr, xq);
3032        }
3033        return dot_q4t_row_avx2(bytes, r, gpr, xq);
3034    }
3035    let mut acc = 0f32;
3036    for gi in 0..gpr {
3037        let tile = &bytes[(r * gpr + gi) * Q4_TILE..(r * gpr + gi + 1) * Q4_TILE];
3038        let s = f16_to_f32(u16::from_le_bytes([tile[0], tile[1]]));
3039        let mut d = 0i32;
3040        for (k, &b) in tile[2..].iter().enumerate() {
3041            d += ((b & 0x0F) as i32 - 8) * xq[gi * GROUP_SIZE + k * 2] as i32
3042                + (((b >> 4) & 0x0F) as i32 - 8) * xq[gi * GROUP_SIZE + k * 2 + 1] as i32;
3043        }
3044        acc += d as f32 * s;
3045    }
3046    acc
3047}
3048
3049#[cfg(target_arch = "aarch64")]
3050#[target_feature(enable = "neon,dotprod")]
3051unsafe fn dot_q4t_row_sdot(bytes: &[u8], r: usize, gpr: usize, xq: &[i8]) -> f32 {
3052    // SAFETY: callers uphold slice-length contracts (18B tile per group,
3053    // xq.len() == gpr·GROUP_SIZE).
3054    unsafe {
3055        use core::arch::aarch64::*;
3056        use core::arch::asm;
3057        let lomask = vdupq_n_u8(0x0F);
3058        let eight = vdupq_n_s8(8);
3059        let mut acc = 0f32;
3060        for gi in 0..gpr {
3061            let t = bytes.as_ptr().add((r * gpr + gi) * Q4_TILE);
3062            let s = f16_to_f32(u16::from_le_bytes([*t, *t.add(1)]));
3063            let b = vld1q_u8(t.add(2));
3064            let lo = vandq_u8(b, lomask);
3065            let hi = vshrq_n_u8::<4>(b);
3066            let e0 = vsubq_s8(vreinterpretq_s8_u8(vzip1q_u8(lo, hi)), eight);
3067            let e1 = vsubq_s8(vreinterpretq_s8_u8(vzip2q_u8(lo, hi)), eight);
3068            let x0 = vld1q_s8(xq.as_ptr().add(gi * GROUP_SIZE));
3069            let x1 = vld1q_s8(xq.as_ptr().add(gi * GROUP_SIZE + 16));
3070            let (mut a0, mut a1) = (vdupq_n_s32(0), vdupq_n_s32(0));
3071            asm!(
3072                "sdot {a0:v}.4s, {e0:v}.16b, {x0:v}.16b",
3073                "sdot {a1:v}.4s, {e1:v}.16b, {x1:v}.16b",
3074                a0 = inout(vreg) a0, a1 = inout(vreg) a1,
3075                e0 = in(vreg) e0, x0 = in(vreg) x0, e1 = in(vreg) e1, x1 = in(vreg) x1,
3076                options(pure, nomem, nostack),
3077            );
3078            acc += vaddvq_s32(vaddq_s32(a0, a1)) as f32 * s;
3079        }
3080        acc
3081    }
3082}
3083
3084#[cfg(target_arch = "x86_64")]
3085#[target_feature(enable = "avx2")]
3086unsafe fn dot_q4t_row_avx2(bytes: &[u8], r: usize, gpr: usize, xq: &[i8]) -> f32 {
3087    // SAFETY: see dot_q4t_row_sdot.
3088    unsafe {
3089        use core::arch::x86_64::*;
3090        let lomask = _mm_set1_epi8(0x0F);
3091        let eight = _mm256_set1_epi8(8);
3092        let ones = _mm256_set1_epi16(1);
3093        let mut acc = 0f32;
3094        for gi in 0..gpr {
3095            let t = bytes.as_ptr().add((r * gpr + gi) * Q4_TILE);
3096            let s = f16_to_f32(u16::from_le_bytes([*t, *t.add(1)]));
3097            let b = _mm_loadu_si128(t.add(2) as *const __m128i);
3098            let lo = _mm_and_si128(b, lomask);
3099            let hi = _mm_and_si128(_mm_srli_epi16::<4>(b), lomask);
3100            let w = _mm256_sub_epi8(
3101                _mm256_set_m128i(_mm_unpackhi_epi8(lo, hi), _mm_unpacklo_epi8(lo, hi)),
3102                eight,
3103            );
3104            let x = _mm256_loadu_si256(xq.as_ptr().add(gi * GROUP_SIZE) as *const __m256i);
3105            let p16 = _mm256_maddubs_epi16(_mm256_abs_epi8(w), _mm256_sign_epi8(x, w));
3106            let d = _mm256_madd_epi16(p16, ones);
3107            let hi128 = _mm256_extracti128_si256::<1>(d);
3108            let s128 = _mm_add_epi32(_mm256_castsi256_si128(d), hi128);
3109            let s64 = _mm_add_epi32(s128, _mm_srli_si128::<8>(s128));
3110            let s32 = _mm_add_epi32(s64, _mm_srli_si128::<4>(s64));
3111            acc += _mm_cvtsi128_si32(s32) as f32 * s;
3112        }
3113        acc
3114    }
3115}
3116
3117/// VNNI twin of `dot_q4t_row_avx2`: same unpack, `vpdpbusd` replaces
3118/// the maddubs+madd pair (see `dpbusd_hsum` — sums are bit-identical).
3119/// 256-bit VL encoding, so the VEX `vpsignb` stays usable.
3120#[cfg(target_arch = "x86_64")]
3121#[target_feature(enable = "avx2,avx512f,avx512bw,avx512vl,avx512vnni")]
3122unsafe fn dot_q4t_row_vnni(bytes: &[u8], r: usize, gpr: usize, xq: &[i8]) -> f32 {
3123    // SAFETY: see dot_q4t_row_sdot.
3124    unsafe {
3125        use core::arch::x86_64::*;
3126        let lomask = _mm_set1_epi8(0x0F);
3127        let eight = _mm256_set1_epi8(8);
3128        let mut acc = 0f32;
3129        for gi in 0..gpr {
3130            let t = bytes.as_ptr().add((r * gpr + gi) * Q4_TILE);
3131            let s = f16_to_f32(u16::from_le_bytes([*t, *t.add(1)]));
3132            let b = _mm_loadu_si128(t.add(2) as *const __m128i);
3133            let lo = _mm_and_si128(b, lomask);
3134            let hi = _mm_and_si128(_mm_srli_epi16::<4>(b), lomask);
3135            let w = _mm256_sub_epi8(
3136                _mm256_set_m128i(_mm_unpackhi_epi8(lo, hi), _mm_unpacklo_epi8(lo, hi)),
3137                eight,
3138            );
3139            let x = _mm256_loadu_si256(xq.as_ptr().add(gi * GROUP_SIZE) as *const __m256i);
3140            let d = dpbusd_hsum(_mm256_abs_epi8(w), _mm256_sign_epi8(x, w));
3141            acc += d as f32 * s;
3142        }
3143        acc
3144    }
3145}
3146
3147/// One q4_tiled row against FOUR activation streams: the nibble unpack
3148/// and abs() happen once per group instead of once per (group,
3149/// activation) — the unpack is the dominant per-element cost of the
3150/// tiled format (roadmap P0 portable blocking, q4t leg).
3151#[cfg(target_arch = "x86_64")]
3152#[target_feature(enable = "avx2")]
3153unsafe fn dot_q4t_row_1x4_avx2(bytes: &[u8], r: usize, gpr: usize, xs: [&[i8]; 4]) -> [f32; 4] {
3154    // SAFETY: callers uphold the 18B-tile and xq-length contracts.
3155    unsafe {
3156        use core::arch::x86_64::*;
3157        let lomask = _mm_set1_epi8(0x0F);
3158        let eight = _mm256_set1_epi8(8);
3159        let ones = _mm256_set1_epi16(1);
3160        let mut acc = [0f32; 4];
3161        for gi in 0..gpr {
3162            let t = bytes.as_ptr().add((r * gpr + gi) * Q4_TILE);
3163            let s = f16_to_f32(u16::from_le_bytes([*t, *t.add(1)]));
3164            let bb = _mm_loadu_si128(t.add(2) as *const __m128i);
3165            let lo = _mm_and_si128(bb, lomask);
3166            let hi = _mm_and_si128(_mm_srli_epi16::<4>(bb), lomask);
3167            let w = _mm256_sub_epi8(
3168                _mm256_set_m128i(_mm_unpackhi_epi8(lo, hi), _mm_unpacklo_epi8(lo, hi)),
3169                eight,
3170            );
3171            let aw = _mm256_abs_epi8(w);
3172            for (k, xq) in xs.iter().enumerate() {
3173                let x = _mm256_loadu_si256(xq.as_ptr().add(gi * GROUP_SIZE) as *const __m256i);
3174                let p16 = _mm256_maddubs_epi16(aw, _mm256_sign_epi8(x, w));
3175                let d = _mm256_madd_epi16(p16, ones);
3176                let hi128 = _mm256_extracti128_si256::<1>(d);
3177                let s128 = _mm_add_epi32(_mm256_castsi256_si128(d), hi128);
3178                let s64 = _mm_add_epi32(s128, _mm_srli_si128::<8>(s128));
3179                let s32 = _mm_add_epi32(s64, _mm_srli_si128::<4>(s64));
3180                acc[k] += _mm_cvtsi128_si32(s32) as f32 * s;
3181            }
3182        }
3183        acc
3184    }
3185}
3186
3187/// VNNI twin of `dot_q4t_row_1x4_avx2` (see `dpbusd_hsum`).
3188#[cfg(target_arch = "x86_64")]
3189#[target_feature(enable = "avx2,avx512f,avx512bw,avx512vl,avx512vnni")]
3190unsafe fn dot_q4t_row_1x4_vnni(bytes: &[u8], r: usize, gpr: usize, xs: [&[i8]; 4]) -> [f32; 4] {
3191    // SAFETY: callers uphold the 18B-tile and xq-length contracts.
3192    unsafe {
3193        use core::arch::x86_64::*;
3194        let lomask = _mm_set1_epi8(0x0F);
3195        let eight = _mm256_set1_epi8(8);
3196        let mut acc = [0f32; 4];
3197        for gi in 0..gpr {
3198            let t = bytes.as_ptr().add((r * gpr + gi) * Q4_TILE);
3199            let s = f16_to_f32(u16::from_le_bytes([*t, *t.add(1)]));
3200            let bb = _mm_loadu_si128(t.add(2) as *const __m128i);
3201            let lo = _mm_and_si128(bb, lomask);
3202            let hi = _mm_and_si128(_mm_srli_epi16::<4>(bb), lomask);
3203            let w = _mm256_sub_epi8(
3204                _mm256_set_m128i(_mm_unpackhi_epi8(lo, hi), _mm_unpacklo_epi8(lo, hi)),
3205                eight,
3206            );
3207            let aw = _mm256_abs_epi8(w);
3208            for (k, xq) in xs.iter().enumerate() {
3209                let x = _mm256_loadu_si256(xq.as_ptr().add(gi * GROUP_SIZE) as *const __m256i);
3210                let d = dpbusd_hsum(aw, _mm256_sign_epi8(x, w));
3211                acc[k] += d as f32 * s;
3212            }
3213        }
3214        acc
3215    }
3216}
3217
3218/// ARM twin of `dot_q4t_row_1x4_avx2`: one nibble unpack per group
3219/// serves FOUR activation streams. Per stream the group order and f32
3220/// accumulation match `dot_q4t_row_sdot` exactly — batch == matvec
3221/// bit-for-bit.
3222#[cfg(target_arch = "aarch64")]
3223#[target_feature(enable = "neon,dotprod")]
3224unsafe fn dot_q4t_row_1x4_sdot(bytes: &[u8], r: usize, gpr: usize, xs: [&[i8]; 4]) -> [f32; 4] {
3225    // SAFETY: callers uphold the 18B-tile and xq-length contracts.
3226    unsafe {
3227        use core::arch::aarch64::*;
3228        use core::arch::asm;
3229        let lomask = vdupq_n_u8(0x0F);
3230        let eight = vdupq_n_s8(8);
3231        let mut acc = [0f32; 4];
3232        for gi in 0..gpr {
3233            let t = bytes.as_ptr().add((r * gpr + gi) * Q4_TILE);
3234            let s = f16_to_f32(u16::from_le_bytes([*t, *t.add(1)]));
3235            let b = vld1q_u8(t.add(2));
3236            let lo = vandq_u8(b, lomask);
3237            let hi = vshrq_n_u8::<4>(b);
3238            let e0 = vsubq_s8(vreinterpretq_s8_u8(vzip1q_u8(lo, hi)), eight);
3239            let e1 = vsubq_s8(vreinterpretq_s8_u8(vzip2q_u8(lo, hi)), eight);
3240            for (k, xq) in xs.iter().enumerate() {
3241                let x0 = vld1q_s8(xq.as_ptr().add(gi * GROUP_SIZE));
3242                let x1 = vld1q_s8(xq.as_ptr().add(gi * GROUP_SIZE + 16));
3243                let (mut a0, mut a1) = (vdupq_n_s32(0), vdupq_n_s32(0));
3244                asm!(
3245                    "sdot {a0:v}.4s, {e0:v}.16b, {x0:v}.16b",
3246                    "sdot {a1:v}.4s, {e1:v}.16b, {x1:v}.16b",
3247                    a0 = inout(vreg) a0, a1 = inout(vreg) a1,
3248                    e0 = in(vreg) e0, x0 = in(vreg) x0, e1 = in(vreg) e1, x1 = in(vreg) x1,
3249                    options(pure, nomem, nostack),
3250                );
3251                acc[k] += vaddvq_s32(vaddq_s32(a0, a1)) as f32 * s;
3252            }
3253        }
3254        acc
3255    }
3256}
3257
3258/// Exact-term correction for A8W8 outliers on a tiled row.
3259#[inline]
3260fn q4t_outlier(bytes: &[u8], r: usize, gpr: usize, j: usize) -> (f32, f32) {
3261    let gi = j / GROUP_SIZE;
3262    let k = j % GROUP_SIZE;
3263    let tile = &bytes[(r * gpr + gi) * Q4_TILE..(r * gpr + gi + 1) * Q4_TILE];
3264    let s = f16_to_f32(u16::from_le_bytes([tile[0], tile[1]]));
3265    let byte = tile[2 + k / 2];
3266    let nib = if k & 1 == 0 { byte & 0x0F } else { byte >> 4 };
3267    ((nib as i32 - 8) as f32, s)
3268}
3269
3270/// Exact scalar q4_tiled row (CMF_SDOT=0 contract) — same pairwise
3271/// accumulation shape as `q4_range_f32`.
3272#[inline]
3273fn q4t_row_exact(bytes: &[u8], r: usize, gpr: usize, x: &[f32]) -> f32 {
3274    let mut acc = 0f32;
3275    for gi in 0..gpr {
3276        let tile = &bytes[(r * gpr + gi) * Q4_TILE..(r * gpr + gi + 1) * Q4_TILE];
3277        let s = f16_to_f32(u16::from_le_bytes([tile[0], tile[1]]));
3278        let xg = &x[gi * GROUP_SIZE..(gi + 1) * GROUP_SIZE];
3279        let mut ga = 0f32;
3280        for (k, &b) in tile[2..].iter().enumerate() {
3281            ga += ((b & 0x0F) as f32 - 8.0) * xg[k * 2]
3282                + (((b >> 4) & 0x0F) as f32 - 8.0) * xg[k * 2 + 1];
3283        }
3284        acc += ga * s;
3285    }
3286    acc
3287}
3288
3289/// Fused q4_tiled matvec (dispatch mirrors `q4matvec`).
3290fn q4t_matvec(
3291    bytes: &[u8],
3292    x: &[f32],
3293    rows: usize,
3294    cols: usize,
3295    out: &mut [f32],
3296    pool: Option<&Pool>,
3297) {
3298    debug_assert_eq!(out.len(), rows);
3299    let gpr = cols / GROUP_SIZE;
3300    let out_addr = SendMut(out.as_mut_ptr());
3301    if a8w8_enabled() {
3302        let act = split_act(x);
3303        let run = move |start: usize, end: usize| {
3304            for r in start..end {
3305                let mut acc = dot_q4t_row_i8(bytes, r, gpr, &act.xq) * act.sx;
3306                for &(j, xv) in &act.outliers {
3307                    let (w, s) = q4t_outlier(bytes, r, gpr, j);
3308                    acc += w * s * xv;
3309                }
3310                // SAFETY: disjoint row ranges per worker.
3311                unsafe { *out_addr.at(r) = acc };
3312            }
3313        };
3314        dispatch_rows(pool, rows, &run);
3315        return;
3316    }
3317    let run = move |start: usize, end: usize| {
3318        for r in start..end {
3319            // SAFETY: disjoint row ranges per worker.
3320            unsafe { *out_addr.at(r) = q4t_row_exact(bytes, r, gpr, x) };
3321        }
3322    };
3323    dispatch_rows(pool, rows, &run);
3324}
3325
3326/// Fused two-input q4_tiled matvec (weights read once per pair).
3327#[allow(clippy::too_many_arguments)]
3328fn q4t_matvec2(
3329    bytes: &[u8],
3330    x1: &[f32],
3331    x2: &[f32],
3332    rows: usize,
3333    cols: usize,
3334    o1: &mut [f32],
3335    o2: &mut [f32],
3336    pool: Option<&Pool>,
3337) {
3338    let gpr = cols / GROUP_SIZE;
3339    let p1 = SendMut(o1.as_mut_ptr());
3340    let p2 = SendMut(o2.as_mut_ptr());
3341    if a8w8_enabled() {
3342        let a1 = split_act(x1);
3343        let a2 = split_act(x2);
3344        let run = move |start: usize, end: usize| {
3345            for r in start..end {
3346                let mut v1 = dot_q4t_row_i8(bytes, r, gpr, &a1.xq) * a1.sx;
3347                let mut v2 = dot_q4t_row_i8(bytes, r, gpr, &a2.xq) * a2.sx;
3348                for &(j, xv) in &a1.outliers {
3349                    let (w, s) = q4t_outlier(bytes, r, gpr, j);
3350                    v1 += w * s * xv;
3351                }
3352                for &(j, xv) in &a2.outliers {
3353                    let (w, s) = q4t_outlier(bytes, r, gpr, j);
3354                    v2 += w * s * xv;
3355                }
3356                // SAFETY: disjoint row ranges per worker.
3357                unsafe {
3358                    *p1.at(r) = v1;
3359                    *p2.at(r) = v2;
3360                }
3361            }
3362        };
3363        dispatch_rows(pool, rows, &run);
3364        return;
3365    }
3366    let run = move |start: usize, end: usize| {
3367        for r in start..end {
3368            // SAFETY: disjoint row ranges per worker.
3369            unsafe {
3370                *p1.at(r) = q4t_row_exact(bytes, r, gpr, x1);
3371                *p2.at(r) = q4t_row_exact(bytes, r, gpr, x2);
3372            }
3373        }
3374    };
3375    dispatch_rows(pool, rows, &run);
3376}
3377
3378/// Batched q4_tiled matmat: each row's tiles stream once per microbatch.
3379#[allow(clippy::too_many_arguments)]
3380/// Prefill GEMM through Accelerate for group-quantized codecs: a
3381/// caller-supplied row dequantizer fills f32 tiles (pool-parallel) and
3382/// each tile rides the AMX with one sgemm — the generic sibling of
3383/// `qmatmat_accel` (q8). Numerics are f32-GEMM (tolerance class);
3384/// decode (b=1) never takes this path.
3385#[cfg(target_os = "macos")]
3386fn dequant_matmat_accel(
3387    dequant_row: &(dyn Fn(usize, &mut [f32]) + Sync),
3388    xs_all: &[f32],
3389    b: usize,
3390    rows: usize,
3391    cols: usize,
3392    out: &mut [f32],
3393    pool: Option<&Pool>,
3394) {
3395    const TR: usize = 2048;
3396    thread_local! {
3397        static WTILE: std::cell::RefCell<Vec<f32>> = const { std::cell::RefCell::new(Vec::new()) };
3398    }
3399    WTILE.with(|wt| {
3400        let mut wtile = wt.borrow_mut();
3401        wtile.resize(TR * cols, 0.0);
3402        let mut r0 = 0usize;
3403        while r0 < rows {
3404            let tr = TR.min(rows - r0);
3405            let wt_addr = SendMut(wtile.as_mut_ptr());
3406            let run = |start: usize, end: usize| {
3407                for r in start..end {
3408                    // SAFETY: workers cover disjoint r ranges.
3409                    let dst = unsafe { std::slice::from_raw_parts_mut(wt_addr.at(r * cols), cols) };
3410                    dequant_row(r0 + r, dst);
3411                }
3412            };
3413            dispatch_rows(pool, tr, &run);
3414            unsafe {
3415                accel_blas::cblas_sgemm(
3416                    101, // RowMajor
3417                    111, // NoTrans A
3418                    112, // Trans B
3419                    b as i32,
3420                    tr as i32,
3421                    cols as i32,
3422                    1.0,
3423                    xs_all.as_ptr(),
3424                    cols as i32,
3425                    wtile.as_ptr(),
3426                    cols as i32,
3427                    0.0,
3428                    out.as_mut_ptr().add(r0),
3429                    rows as i32,
3430                );
3431            }
3432            r0 += tr;
3433        }
3434    });
3435}
3436
3437fn q4t_matmat(
3438    bytes: &[u8],
3439    xs_all: &[f32],
3440    b: usize,
3441    rows: usize,
3442    cols: usize,
3443    out: &mut [f32],
3444    pool: Option<&Pool>,
3445) {
3446    debug_assert_eq!(out.len(), b * rows);
3447    let gpr = cols / GROUP_SIZE;
3448    // Wide batches ride the AMX like q8's qmatmat: on Apple silicon
3449    // the dequant-tile sgemm is an order above the SDOT row loop for
3450    // prefill shapes (imagegen DiT forwards are exactly this).
3451    #[cfg(target_os = "macos")]
3452    if b >= 8 && rows * cols >= 500_000 && accel_gemm_enabled() {
3453        dequant_matmat_accel(
3454            &|r, dst| {
3455                for gi in 0..gpr {
3456                    let tile = &bytes[(r * gpr + gi) * Q4_TILE..(r * gpr + gi + 1) * Q4_TILE];
3457                    let s = f16_to_f32(u16::from_le_bytes([tile[0], tile[1]]));
3458                    for (k, &bb) in tile[2..].iter().enumerate() {
3459                        dst[gi * GROUP_SIZE + k * 2] = ((bb & 0x0F) as f32 - 8.0) * s;
3460                        dst[gi * GROUP_SIZE + k * 2 + 1] = (((bb >> 4) & 0x0F) as f32 - 8.0) * s;
3461                    }
3462                }
3463            },
3464            xs_all,
3465            b,
3466            rows,
3467            cols,
3468            out,
3469            pool,
3470        );
3471        return;
3472    }
3473    let out_addr = SendMut(out.as_mut_ptr());
3474    if a8w8_enabled() {
3475        let acts: Vec<SplitAct> = (0..b)
3476            .map(|bi| split_act(&xs_all[bi * cols..(bi + 1) * cols]))
3477            .collect();
3478        let acts = &acts;
3479        #[cfg(target_arch = "x86_64")]
3480        let blocked_ok = avx2_enabled()
3481            && std::env::var("CMF_X86_BLOCKED")
3482                .map(|v| v != "0")
3483                .unwrap_or(true);
3484        #[cfg(target_arch = "aarch64")]
3485        let blocked_ok = sdot_enabled()
3486            && std::env::var("CMF_X86_BLOCKED")
3487                .map(|v| v != "0")
3488                .unwrap_or(true);
3489        #[cfg(not(any(target_arch = "x86_64", target_arch = "aarch64")))]
3490        let blocked_ok = false;
3491        let run = move |start: usize, end: usize| {
3492            for r in start..end {
3493                let mut bi = 0usize;
3494                #[cfg(target_arch = "aarch64")]
3495                if blocked_ok {
3496                    while bi + 4 <= acts.len() {
3497                        let xs = [
3498                            acts[bi].xq.as_slice(),
3499                            acts[bi + 1].xq.as_slice(),
3500                            acts[bi + 2].xq.as_slice(),
3501                            acts[bi + 3].xq.as_slice(),
3502                        ];
3503                        let d = unsafe { dot_q4t_row_1x4_sdot(bytes, r, gpr, xs) };
3504                        for k in 0..4 {
3505                            let act = &acts[bi + k];
3506                            let mut acc = d[k] * act.sx;
3507                            for &(j, xv) in &act.outliers {
3508                                let (w, sc) = q4t_outlier(bytes, r, gpr, j);
3509                                acc += w * sc * xv;
3510                            }
3511                            // SAFETY: disjoint (bi, r) cells per worker.
3512                            unsafe { *out_addr.at((bi + k) * rows + r) = acc };
3513                        }
3514                        bi += 4;
3515                    }
3516                }
3517                #[cfg(target_arch = "x86_64")]
3518                if blocked_ok {
3519                    while bi + 4 <= acts.len() {
3520                        let xs = [
3521                            acts[bi].xq.as_slice(),
3522                            acts[bi + 1].xq.as_slice(),
3523                            acts[bi + 2].xq.as_slice(),
3524                            acts[bi + 3].xq.as_slice(),
3525                        ];
3526                        let d = unsafe {
3527                            if vnni_tiles_enabled() {
3528                                dot_q4t_row_1x4_vnni(bytes, r, gpr, xs)
3529                            } else {
3530                                dot_q4t_row_1x4_avx2(bytes, r, gpr, xs)
3531                            }
3532                        };
3533                        for k in 0..4 {
3534                            let act = &acts[bi + k];
3535                            let mut acc = d[k] * act.sx;
3536                            for &(j, xv) in &act.outliers {
3537                                let (w, sc) = q4t_outlier(bytes, r, gpr, j);
3538                                acc += w * sc * xv;
3539                            }
3540                            // SAFETY: disjoint (bi, r) cells per worker.
3541                            unsafe { *out_addr.at((bi + k) * rows + r) = acc };
3542                        }
3543                        bi += 4;
3544                    }
3545                }
3546                let _ = blocked_ok;
3547                while bi < acts.len() {
3548                    let act = &acts[bi];
3549                    let mut acc = dot_q4t_row_i8(bytes, r, gpr, &act.xq) * act.sx;
3550                    for &(j, xv) in &act.outliers {
3551                        let (w, s) = q4t_outlier(bytes, r, gpr, j);
3552                        acc += w * s * xv;
3553                    }
3554                    // SAFETY: disjoint (bi, r) cells per worker range.
3555                    unsafe { *out_addr.at(bi * rows + r) = acc };
3556                    bi += 1;
3557                }
3558            }
3559        };
3560        dispatch_rows(pool, rows, &run);
3561        return;
3562    }
3563    let run = move |start: usize, end: usize| {
3564        for r in start..end {
3565            for bi in 0..b {
3566                let x = &xs_all[bi * cols..(bi + 1) * cols];
3567                // SAFETY: disjoint (bi, r) cells per worker range.
3568                unsafe { *out_addr.at(bi * rows + r) = q4t_row_exact(bytes, r, gpr, x) };
3569            }
3570        }
3571    };
3572    dispatch_rows(pool, rows, &run);
3573}
3574
3575// ── q1 (dtype 12): binary weights, [f16 scale][4B sign bits] per
3576// 32-group tile. The kernel family mirrors q4_tiled: one sequential
3577// stream of 6-byte tiles, per-tile integer dot × scale, exact outlier
3578// correction (A8W8 contract), exact scalar path under CMF_SDOT=0. ──
3579
3580/// Per-32-group sums of the quantized activation — the ±1 identity's
3581/// shared half: `dot = −2·sdot(mask, x) − gsum[g]`, computed ONCE per
3582/// matvec and reused by every row.
3583fn q1_group_sums(xq: &[i8], gpr: usize) -> Vec<i32> {
3584    (0..gpr)
3585        .map(|gi| {
3586            xq[gi * GROUP_SIZE..(gi + 1) * GROUP_SIZE]
3587                .iter()
3588                .map(|&v| v as i32)
3589                .sum()
3590        })
3591        .collect()
3592}
3593
3594/// One q1 row via the A8W8 int8 path — mask-SDOT on ARM (no ±1
3595/// expansion at all), scalar bit loop elsewhere (AVX2 queued with the
3596/// x86 pass).
3597#[inline]
3598#[allow(unreachable_code)]
3599/// AVX2 q1 row via the same ±1 identity as the ARM sdot kernel: the
3600/// sign bits expand to a {0, −1} byte mask through shuffle+cmpeq, the
3601/// masked activation sums through maddubs(1, x&mask), and
3602/// `dot = −(2·masked_sum + Σx_group)` — bit-identical integer math.
3603#[cfg(target_arch = "x86_64")]
3604#[target_feature(enable = "avx2")]
3605unsafe fn dot_q1_row_avx2(bytes: &[u8], r: usize, gpr: usize, xq: &[i8], gsum: &[i32]) -> f32 {
3606    // SAFETY: callers uphold the 6B-tile and xq/gsum length contracts.
3607    unsafe {
3608        use core::arch::x86_64::*;
3609        // Byte j of the mask must replicate bits-byte j/8.
3610        let expand = _mm256_setr_epi8(
3611            0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3,
3612            3, 3, 3,
3613        );
3614        let bitsel = _mm256_setr_epi8(
3615            1, 2, 4, 8, 16, 32, 64, -128, 1, 2, 4, 8, 16, 32, 64, -128, 1, 2, 4, 8, 16, 32, 64,
3616            -128, 1, 2, 4, 8, 16, 32, 64, -128,
3617        );
3618        let ones8 = _mm256_set1_epi8(1);
3619        let ones16 = _mm256_set1_epi16(1);
3620        let mut acc = 0f32;
3621        for gi in 0..gpr {
3622            let t = bytes.as_ptr().add((r * gpr + gi) * Q1_TILE);
3623            let s = f16_to_f32(u16::from_le_bytes([*t, *t.add(1)]));
3624            let bits = u32::from_le_bytes([*t.add(2), *t.add(3), *t.add(4), *t.add(5)]);
3625            let bc = _mm256_shuffle_epi8(_mm256_set1_epi32(bits as i32), expand);
3626            let mask = _mm256_cmpeq_epi8(_mm256_and_si256(bc, bitsel), bitsel);
3627            let x = _mm256_loadu_si256(xq.as_ptr().add(gi * GROUP_SIZE) as *const __m256i);
3628            let sel = _mm256_and_si256(x, mask);
3629            // Σ of selected i8 lanes: maddubs(1u8, sel_i8) pairs → madd.
3630            let p16 = _mm256_maddubs_epi16(ones8, sel);
3631            let d32 = _mm256_madd_epi16(p16, ones16);
3632            let hi128 = _mm256_extracti128_si256::<1>(d32);
3633            let s128 = _mm_add_epi32(_mm256_castsi256_si128(d32), hi128);
3634            let s64 = _mm_add_epi32(s128, _mm_srli_si128::<8>(s128));
3635            let s32 = _mm_add_epi32(s64, _mm_srli_si128::<4>(s64));
3636            let msum = _mm_cvtsi128_si32(s32);
3637            // The and-select keeps x UN-negated (unlike ARM's −1-mask
3638            // sdot): d = Σ_set − Σ_unset = 2·Σ_set − Σ_all.
3639            let d = 2 * msum - gsum[gi];
3640            acc += d as f32 * s;
3641        }
3642        acc
3643    }
3644}
3645
3646/// VNNI twin of `dot_q1_row_avx2`: the masked-select sum goes through
3647/// one `vpdpbusd(1u8, sel)` (see `dpbusd_hsum` — bit-identical).
3648#[cfg(target_arch = "x86_64")]
3649#[target_feature(enable = "avx2,avx512f,avx512bw,avx512vl,avx512vnni")]
3650unsafe fn dot_q1_row_vnni(bytes: &[u8], r: usize, gpr: usize, xq: &[i8], gsum: &[i32]) -> f32 {
3651    // SAFETY: callers uphold the 6B-tile and xq/gsum length contracts.
3652    unsafe {
3653        use core::arch::x86_64::*;
3654        let expand = _mm256_setr_epi8(
3655            0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3,
3656            3, 3, 3,
3657        );
3658        let bitsel = _mm256_setr_epi8(
3659            1, 2, 4, 8, 16, 32, 64, -128, 1, 2, 4, 8, 16, 32, 64, -128, 1, 2, 4, 8, 16, 32, 64,
3660            -128, 1, 2, 4, 8, 16, 32, 64, -128,
3661        );
3662        let ones8 = _mm256_set1_epi8(1);
3663        let mut acc = 0f32;
3664        for gi in 0..gpr {
3665            let t = bytes.as_ptr().add((r * gpr + gi) * Q1_TILE);
3666            let s = f16_to_f32(u16::from_le_bytes([*t, *t.add(1)]));
3667            let bits = u32::from_le_bytes([*t.add(2), *t.add(3), *t.add(4), *t.add(5)]);
3668            let bc = _mm256_shuffle_epi8(_mm256_set1_epi32(bits as i32), expand);
3669            let mask = _mm256_cmpeq_epi8(_mm256_and_si256(bc, bitsel), bitsel);
3670            let x = _mm256_loadu_si256(xq.as_ptr().add(gi * GROUP_SIZE) as *const __m256i);
3671            let msum = dpbusd_hsum(ones8, _mm256_and_si256(x, mask));
3672            let d = 2 * msum - gsum[gi];
3673            acc += d as f32 * s;
3674        }
3675        acc
3676    }
3677}
3678
3679/// VNNI twin of `dot_q1_row_1x4_avx2` (see `dpbusd_hsum`).
3680#[cfg(target_arch = "x86_64")]
3681#[target_feature(enable = "avx2,avx512f,avx512bw,avx512vl,avx512vnni")]
3682unsafe fn dot_q1_row_1x4_vnni(
3683    bytes: &[u8],
3684    r: usize,
3685    gpr: usize,
3686    xs: [&[i8]; 4],
3687    gsums: [&[i32]; 4],
3688) -> [f32; 4] {
3689    // SAFETY: callers uphold the 6B-tile and xq/gsum length contracts.
3690    unsafe {
3691        use core::arch::x86_64::*;
3692        let expand = _mm256_setr_epi8(
3693            0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3,
3694            3, 3, 3,
3695        );
3696        let bitsel = _mm256_setr_epi8(
3697            1, 2, 4, 8, 16, 32, 64, -128, 1, 2, 4, 8, 16, 32, 64, -128, 1, 2, 4, 8, 16, 32, 64,
3698            -128, 1, 2, 4, 8, 16, 32, 64, -128,
3699        );
3700        let ones8 = _mm256_set1_epi8(1);
3701        let mut acc = [0f32; 4];
3702        for gi in 0..gpr {
3703            let t = bytes.as_ptr().add((r * gpr + gi) * Q1_TILE);
3704            let s = f16_to_f32(u16::from_le_bytes([*t, *t.add(1)]));
3705            let bits = u32::from_le_bytes([*t.add(2), *t.add(3), *t.add(4), *t.add(5)]);
3706            let bc = _mm256_shuffle_epi8(_mm256_set1_epi32(bits as i32), expand);
3707            let mask = _mm256_cmpeq_epi8(_mm256_and_si256(bc, bitsel), bitsel);
3708            for (k, xq) in xs.iter().enumerate() {
3709                let x = _mm256_loadu_si256(xq.as_ptr().add(gi * GROUP_SIZE) as *const __m256i);
3710                let msum = dpbusd_hsum(ones8, _mm256_and_si256(x, mask));
3711                let d = 2 * msum - gsums[k][gi];
3712                acc[k] += d as f32 * s;
3713            }
3714        }
3715        acc
3716    }
3717}
3718
3719/// The blocked 1×4 flavor: the expanded bit mask serves four activation
3720/// streams per group (mask build once, four select+reduce chains).
3721#[cfg(target_arch = "x86_64")]
3722#[target_feature(enable = "avx2")]
3723unsafe fn dot_q1_row_1x4_avx2(
3724    bytes: &[u8],
3725    r: usize,
3726    gpr: usize,
3727    xs: [&[i8]; 4],
3728    gsums: [&[i32]; 4],
3729) -> [f32; 4] {
3730    // SAFETY: callers uphold the 6B-tile and xq/gsum length contracts.
3731    unsafe {
3732        use core::arch::x86_64::*;
3733        let expand = _mm256_setr_epi8(
3734            0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3,
3735            3, 3, 3,
3736        );
3737        let bitsel = _mm256_setr_epi8(
3738            1, 2, 4, 8, 16, 32, 64, -128, 1, 2, 4, 8, 16, 32, 64, -128, 1, 2, 4, 8, 16, 32, 64,
3739            -128, 1, 2, 4, 8, 16, 32, 64, -128,
3740        );
3741        let ones8 = _mm256_set1_epi8(1);
3742        let ones16 = _mm256_set1_epi16(1);
3743        let mut acc = [0f32; 4];
3744        for gi in 0..gpr {
3745            let t = bytes.as_ptr().add((r * gpr + gi) * Q1_TILE);
3746            let s = f16_to_f32(u16::from_le_bytes([*t, *t.add(1)]));
3747            let bits = u32::from_le_bytes([*t.add(2), *t.add(3), *t.add(4), *t.add(5)]);
3748            let bc = _mm256_shuffle_epi8(_mm256_set1_epi32(bits as i32), expand);
3749            let mask = _mm256_cmpeq_epi8(_mm256_and_si256(bc, bitsel), bitsel);
3750            for (k, xq) in xs.iter().enumerate() {
3751                let x = _mm256_loadu_si256(xq.as_ptr().add(gi * GROUP_SIZE) as *const __m256i);
3752                let sel = _mm256_and_si256(x, mask);
3753                let p16 = _mm256_maddubs_epi16(ones8, sel);
3754                let d32 = _mm256_madd_epi16(p16, ones16);
3755                let hi128 = _mm256_extracti128_si256::<1>(d32);
3756                let s128 = _mm_add_epi32(_mm256_castsi256_si128(d32), hi128);
3757                let s64 = _mm_add_epi32(s128, _mm_srli_si128::<8>(s128));
3758                let s32 = _mm_add_epi32(s64, _mm_srli_si128::<4>(s64));
3759                let msum = _mm_cvtsi128_si32(s32);
3760                let d = 2 * msum - gsums[k][gi];
3761                acc[k] += d as f32 * s;
3762            }
3763        }
3764        acc
3765    }
3766}
3767
3768#[allow(unreachable_code)]
3769fn dot_q1_row_i8(bytes: &[u8], r: usize, gpr: usize, xq: &[i8], gsum: &[i32]) -> f32 {
3770    #[cfg(target_arch = "aarch64")]
3771    unsafe {
3772        return dot_q1_row_sdot(bytes, r, gpr, xq, gsum);
3773    }
3774    #[cfg(target_arch = "x86_64")]
3775    if avx2_enabled() {
3776        unsafe {
3777            if vnni_tiles_enabled() {
3778                return dot_q1_row_vnni(bytes, r, gpr, xq, gsum);
3779            }
3780            return dot_q1_row_avx2(bytes, r, gpr, xq, gsum);
3781        }
3782    }
3783    let _ = gsum;
3784    let mut acc = 0f32;
3785    for gi in 0..gpr {
3786        let tile = &bytes[(r * gpr + gi) * Q1_TILE..(r * gpr + gi + 1) * Q1_TILE];
3787        let s = f16_to_f32(u16::from_le_bytes([tile[0], tile[1]]));
3788        let mut d = 0i32;
3789        for (j, &b) in tile[2..].iter().enumerate() {
3790            for k in 0..8 {
3791                let w = ((b >> k) & 1) as i32 * 2 - 1;
3792                d += w * xq[gi * GROUP_SIZE + j * 8 + k] as i32;
3793            }
3794        }
3795        acc += d as f32 * s;
3796    }
3797    acc
3798}
3799
3800/// SDOT q1 row via the ±1 identity: the vtst mask (0xFF where the bit
3801/// is set, i.e. −1 as i8) feeds `sdot` DIRECTLY — no expansion to ±1
3802/// lanes at all — and `dot = −(2·sdot(mask, x) + Σx_group)`, with the
3803/// per-group activation sums shared across every row of the matvec.
3804/// Four tiles (128 weights) per iteration: integer dots reduce through
3805/// a vpaddq tree into ONE i32x4 that meets its four scales in a single
3806/// fused f32 multiply-add. Integer math throughout — bit-identical to
3807/// the scalar ±1 reference.
3808#[cfg(target_arch = "aarch64")]
3809#[target_feature(enable = "neon,dotprod")]
3810unsafe fn dot_q1_row_sdot(bytes: &[u8], r: usize, gpr: usize, xq: &[i8], gsum: &[i32]) -> f32 {
3811    // SAFETY: callers uphold slice-length contracts (6B tile per group,
3812    // xq.len() == gpr·GROUP_SIZE, gsum.len() == gpr).
3813    unsafe {
3814        use core::arch::aarch64::*;
3815        use core::arch::asm;
3816        const MASKS: [u8; 16] = [1, 2, 4, 8, 16, 32, 64, 128, 1, 2, 4, 8, 16, 32, 64, 128];
3817        let m = vld1q_u8(MASKS.as_ptr());
3818        // One tile's −Σ_set(x) as an UNREDUCED i32x4 (two mask-sdots).
3819        macro_rules! tile_dot {
3820            ($t:expr, $x:expr) => {{
3821                let v0 = vcombine_u8(vdup_n_u8(*$t.add(2)), vdup_n_u8(*$t.add(3)));
3822                let v1 = vcombine_u8(vdup_n_u8(*$t.add(4)), vdup_n_u8(*$t.add(5)));
3823                let w0 = vreinterpretq_s8_u8(vtstq_u8(v0, m));
3824                let w1 = vreinterpretq_s8_u8(vtstq_u8(v1, m));
3825                let x0 = vld1q_s8($x);
3826                let x1 = vld1q_s8($x.add(16));
3827                let (mut a0, mut a1) = (vdupq_n_s32(0), vdupq_n_s32(0));
3828                asm!(
3829                    "sdot {a0:v}.4s, {w0:v}.16b, {x0:v}.16b",
3830                    "sdot {a1:v}.4s, {w1:v}.16b, {x1:v}.16b",
3831                    a0 = inout(vreg) a0, a1 = inout(vreg) a1,
3832                    w0 = in(vreg) w0, x0 = in(vreg) x0, w1 = in(vreg) w1, x1 = in(vreg) x1,
3833                    options(pure, nomem, nostack),
3834                );
3835                vaddq_s32(a0, a1)
3836            }};
3837        }
3838        // TBL unpack over PAIR loads: one vld1q covers two 6B tiles
3839        // ([s s b b b b][s s b b b b] + 4B slack), TBL replicates each
3840        // bit-byte across 8 lanes for vtst, and the four scales gather
3841        // through tbl2 into one fcvtl — the 16 ld1r broadcast loads and
3842        // 4 branchy software f16 conversions per 128 weights (the
3843        // measured load-port wall of this kernel) become 2 vector
3844        // loads + 9 table lookups. Integer math order is unchanged —
3845        // bit-identical results (FCVTL is exact on every f16).
3846        const IW00: [u8; 16] = [2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3];
3847        const IW01: [u8; 16] = [4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5];
3848        const IW10: [u8; 16] = [8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9];
3849        const IW11: [u8; 16] = [
3850            10, 10, 10, 10, 10, 10, 10, 10, 11, 11, 11, 11, 11, 11, 11, 11,
3851        ];
3852        const ISC: [u8; 8] = [0, 1, 6, 7, 16, 17, 22, 23];
3853        let (iw00, iw01) = (vld1q_u8(IW00.as_ptr()), vld1q_u8(IW01.as_ptr()));
3854        let (iw10, iw11) = (vld1q_u8(IW10.as_ptr()), vld1q_u8(IW11.as_ptr()));
3855        let isc = vld1_u8(ISC.as_ptr());
3856        // One tile's −Σ_set(x) from a TBL-unpacked pair load.
3857        macro_rules! tile_dot_tbl {
3858            ($ld:expr, $i0:expr, $i1:expr, $x:expr) => {{
3859                let w0 = vreinterpretq_s8_u8(vtstq_u8(vqtbl1q_u8($ld, $i0), m));
3860                let w1 = vreinterpretq_s8_u8(vtstq_u8(vqtbl1q_u8($ld, $i1), m));
3861                let x0 = vld1q_s8($x);
3862                let x1 = vld1q_s8($x.add(16));
3863                let (mut a0, mut a1) = (vdupq_n_s32(0), vdupq_n_s32(0));
3864                asm!(
3865                    "sdot {a0:v}.4s, {w0:v}.16b, {x0:v}.16b",
3866                    "sdot {a1:v}.4s, {w1:v}.16b, {x1:v}.16b",
3867                    a0 = inout(vreg) a0, a1 = inout(vreg) a1,
3868                    w0 = in(vreg) w0, x0 = in(vreg) x0, w1 = in(vreg) w1, x1 = in(vreg) x1,
3869                    options(pure, nomem, nostack),
3870                );
3871                vaddq_s32(a0, a1)
3872            }};
3873        }
3874        let base = bytes.as_ptr().add(r * gpr * Q1_TILE);
3875        let row_base = r * gpr * Q1_TILE;
3876        let abs_end = bytes.len();
3877        let xp = xq.as_ptr();
3878        let gp = gsum.as_ptr();
3879        let mut accv = vdupq_n_f32(0.0);
3880        let mut gi = 0;
3881        // The second pair load reads 4B past tile gi+3 — stay inside
3882        // the payload slice (only the file's final tiles fall back).
3883        while gi + 4 <= gpr && row_base + (gi + 4) * Q1_TILE + 4 <= abs_end {
3884            let t0 = base.add(gi * Q1_TILE);
3885            let ld_a = vld1q_u8(t0);
3886            let ld_b = vld1q_u8(t0.add(2 * Q1_TILE));
3887            let d0 = tile_dot_tbl!(ld_a, iw00, iw01, xp.add(gi * GROUP_SIZE));
3888            let d1 = tile_dot_tbl!(ld_a, iw10, iw11, xp.add((gi + 1) * GROUP_SIZE));
3889            let d2 = tile_dot_tbl!(ld_b, iw00, iw01, xp.add((gi + 2) * GROUP_SIZE));
3890            let d3 = tile_dot_tbl!(ld_b, iw10, iw11, xp.add((gi + 3) * GROUP_SIZE));
3891            // [−Σ0, −Σ1, −Σ2, −Σ3] → dots = −(2·Σset_neg + gsum)
3892            let neg = vpaddq_s32(vpaddq_s32(d0, d1), vpaddq_s32(d2, d3));
3893            let g = vld1q_s32(gp.add(gi));
3894            let dots = vnegq_s32(vaddq_s32(vshlq_n_s32::<1>(neg), g));
3895            let sc16 = vqtbl2_u8(uint8x16x2_t(ld_a, ld_b), isc);
3896            let scf: float32x4_t;
3897            asm!(
3898                "fcvtl {o:v}.4s, {i:v}.4h",
3899                o = out(vreg) scf, i = in(vreg) sc16,
3900                options(pure, nomem, nostack),
3901            );
3902            accv = vfmaq_f32(accv, vcvtq_f32_s32(dots), scf);
3903            gi += 4;
3904        }
3905        let mut acc = vaddvq_f32(accv);
3906        while gi < gpr {
3907            let t = base.add(gi * Q1_TILE);
3908            let s = f16_to_f32(u16::from_le_bytes([*t, *t.add(1)]));
3909            let d = vaddvq_s32(tile_dot!(t, xp.add(gi * GROUP_SIZE)));
3910            acc += (-(2 * d + *gp.add(gi))) as f32 * s;
3911            gi += 1;
3912        }
3913        acc
3914    }
3915}
3916
3917/// Blocked q1 1×4: one TBL unpack of the tile pair serves FOUR
3918/// activation streams (prefill amortization — the same idea as the
3919/// AVX2 twin; per stream the group order, fma order and tail match the
3920/// single-row kernel exactly, so batch == matvec bit-for-bit).
3921#[cfg(target_arch = "aarch64")]
3922#[target_feature(enable = "neon,dotprod")]
3923unsafe fn dot_q1_row_1x4_sdot(
3924    bytes: &[u8],
3925    r: usize,
3926    gpr: usize,
3927    xs: [&[i8]; 4],
3928    gs: [&[i32]; 4],
3929) -> [f32; 4] {
3930    // SAFETY: same slice-length contracts as `dot_q1_row_sdot`, ×4.
3931    unsafe {
3932        use core::arch::aarch64::*;
3933        use core::arch::asm;
3934        const MASKS: [u8; 16] = [1, 2, 4, 8, 16, 32, 64, 128, 1, 2, 4, 8, 16, 32, 64, 128];
3935        const IW00: [u8; 16] = [2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3];
3936        const IW01: [u8; 16] = [4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5];
3937        const IW10: [u8; 16] = [8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9];
3938        const IW11: [u8; 16] = [
3939            10, 10, 10, 10, 10, 10, 10, 10, 11, 11, 11, 11, 11, 11, 11, 11,
3940        ];
3941        const ISC: [u8; 8] = [0, 1, 6, 7, 16, 17, 22, 23];
3942        let m = vld1q_u8(MASKS.as_ptr());
3943        let (iw00, iw01) = (vld1q_u8(IW00.as_ptr()), vld1q_u8(IW01.as_ptr()));
3944        let (iw10, iw11) = (vld1q_u8(IW10.as_ptr()), vld1q_u8(IW11.as_ptr()));
3945        let isc = vld1_u8(ISC.as_ptr());
3946        macro_rules! sdot2 {
3947            ($w0:expr, $w1:expr, $x:expr) => {{
3948                let x0 = vld1q_s8($x);
3949                let x1 = vld1q_s8($x.add(16));
3950                let (mut a0, mut a1) = (vdupq_n_s32(0), vdupq_n_s32(0));
3951                asm!(
3952                    "sdot {a0:v}.4s, {w0:v}.16b, {x0:v}.16b",
3953                    "sdot {a1:v}.4s, {w1:v}.16b, {x1:v}.16b",
3954                    a0 = inout(vreg) a0, a1 = inout(vreg) a1,
3955                    w0 = in(vreg) $w0, x0 = in(vreg) x0, w1 = in(vreg) $w1, x1 = in(vreg) x1,
3956                    options(pure, nomem, nostack),
3957                );
3958                vaddq_s32(a0, a1)
3959            }};
3960        }
3961        let base = bytes.as_ptr().add(r * gpr * Q1_TILE);
3962        let row_base = r * gpr * Q1_TILE;
3963        let abs_end = bytes.len();
3964        let mut accv = [vdupq_n_f32(0.0); 4];
3965        let mut gi = 0;
3966        while gi + 4 <= gpr && row_base + (gi + 4) * Q1_TILE + 4 <= abs_end {
3967            let t0 = base.add(gi * Q1_TILE);
3968            let ld_a = vld1q_u8(t0);
3969            let ld_b = vld1q_u8(t0.add(2 * Q1_TILE));
3970            // Unpack ONCE — eight ±mask vectors serve all four streams.
3971            let w00 = vreinterpretq_s8_u8(vtstq_u8(vqtbl1q_u8(ld_a, iw00), m));
3972            let w01 = vreinterpretq_s8_u8(vtstq_u8(vqtbl1q_u8(ld_a, iw01), m));
3973            let w10 = vreinterpretq_s8_u8(vtstq_u8(vqtbl1q_u8(ld_a, iw10), m));
3974            let w11 = vreinterpretq_s8_u8(vtstq_u8(vqtbl1q_u8(ld_a, iw11), m));
3975            let w20 = vreinterpretq_s8_u8(vtstq_u8(vqtbl1q_u8(ld_b, iw00), m));
3976            let w21 = vreinterpretq_s8_u8(vtstq_u8(vqtbl1q_u8(ld_b, iw01), m));
3977            let w30 = vreinterpretq_s8_u8(vtstq_u8(vqtbl1q_u8(ld_b, iw10), m));
3978            let w31 = vreinterpretq_s8_u8(vtstq_u8(vqtbl1q_u8(ld_b, iw11), m));
3979            let sc16 = vqtbl2_u8(uint8x16x2_t(ld_a, ld_b), isc);
3980            let scf: float32x4_t;
3981            asm!(
3982                "fcvtl {o:v}.4s, {i:v}.4h",
3983                o = out(vreg) scf, i = in(vreg) sc16,
3984                options(pure, nomem, nostack),
3985            );
3986            for k in 0..4 {
3987                let xp = xs[k].as_ptr();
3988                let d0 = sdot2!(w00, w01, xp.add(gi * GROUP_SIZE));
3989                let d1 = sdot2!(w10, w11, xp.add((gi + 1) * GROUP_SIZE));
3990                let d2 = sdot2!(w20, w21, xp.add((gi + 2) * GROUP_SIZE));
3991                let d3 = sdot2!(w30, w31, xp.add((gi + 3) * GROUP_SIZE));
3992                let neg = vpaddq_s32(vpaddq_s32(d0, d1), vpaddq_s32(d2, d3));
3993                let g = vld1q_s32(gs[k].as_ptr().add(gi));
3994                let dots = vnegq_s32(vaddq_s32(vshlq_n_s32::<1>(neg), g));
3995                accv[k] = vfmaq_f32(accv[k], vcvtq_f32_s32(dots), scf);
3996            }
3997            gi += 4;
3998        }
3999        let mut acc = [
4000            vaddvq_f32(accv[0]),
4001            vaddvq_f32(accv[1]),
4002            vaddvq_f32(accv[2]),
4003            vaddvq_f32(accv[3]),
4004        ];
4005        while gi < gpr {
4006            let t = base.add(gi * Q1_TILE);
4007            let sc = f16_to_f32(u16::from_le_bytes([*t, *t.add(1)]));
4008            let v0 = vcombine_u8(vdup_n_u8(*t.add(2)), vdup_n_u8(*t.add(3)));
4009            let v1 = vcombine_u8(vdup_n_u8(*t.add(4)), vdup_n_u8(*t.add(5)));
4010            let w0 = vreinterpretq_s8_u8(vtstq_u8(v0, m));
4011            let w1 = vreinterpretq_s8_u8(vtstq_u8(v1, m));
4012            for k in 0..4 {
4013                let d = vaddvq_s32(sdot2!(w0, w1, xs[k].as_ptr().add(gi * GROUP_SIZE)));
4014                acc[k] += (-(2 * d + *gs[k].as_ptr().add(gi))) as f32 * sc;
4015            }
4016            gi += 1;
4017        }
4018        acc
4019    }
4020}
4021
4022/// (weight ±1, scale) of one q1 element — the exact outlier term.
4023#[inline]
4024fn q1_outlier(bytes: &[u8], r: usize, gpr: usize, j: usize) -> (f32, f32) {
4025    let gi = j / GROUP_SIZE;
4026    let k = j % GROUP_SIZE;
4027    let tile = &bytes[(r * gpr + gi) * Q1_TILE..(r * gpr + gi + 1) * Q1_TILE];
4028    let s = f16_to_f32(u16::from_le_bytes([tile[0], tile[1]]));
4029    let bit = (tile[2 + k / 8] >> (k % 8)) & 1;
4030    ((bit as i32 * 2 - 1) as f32, s)
4031}
4032
4033/// Exact scalar q1 row (CMF_SDOT=0 contract).
4034#[inline]
4035fn q1_row_exact(bytes: &[u8], r: usize, gpr: usize, x: &[f32]) -> f32 {
4036    let mut acc = 0f32;
4037    for gi in 0..gpr {
4038        let tile = &bytes[(r * gpr + gi) * Q1_TILE..(r * gpr + gi + 1) * Q1_TILE];
4039        let s = f16_to_f32(u16::from_le_bytes([tile[0], tile[1]]));
4040        let xg = &x[gi * GROUP_SIZE..(gi + 1) * GROUP_SIZE];
4041        let mut ga = 0f32;
4042        for (j, &b) in tile[2..].iter().enumerate() {
4043            for k in 0..8 {
4044                ga += (((b >> k) & 1) as f32 * 2.0 - 1.0) * xg[j * 8 + k];
4045            }
4046        }
4047        acc += ga * s;
4048    }
4049    acc
4050}
4051
4052/// One q1 row range via A8W8 (the body of `q1_matvec`'s hot loop,
4053/// extracted so multi-matrix jobs drive the same kernel).
4054#[allow(clippy::too_many_arguments)]
4055fn q1_range_a8w8(
4056    bytes: &[u8],
4057    gpr: usize,
4058    act: &SplitAct,
4059    gsum: &[i32],
4060    out: SendMut,
4061    start: usize,
4062    end: usize,
4063) {
4064    for r in start..end {
4065        let mut acc = dot_q1_row_i8(bytes, r, gpr, &act.xq, gsum) * act.sx;
4066        for &(j, xv) in &act.outliers {
4067            let (w, s) = q1_outlier(bytes, r, gpr, j);
4068            acc += w * s * xv;
4069        }
4070        // SAFETY: disjoint row ranges per worker.
4071        unsafe { *out.at(r) = acc };
4072    }
4073}
4074
4075/// Exact-scalar q1 row range (CMF_SDOT=0 contract).
4076fn q1_range_f32(bytes: &[u8], gpr: usize, x: &[f32], out: SendMut, start: usize, end: usize) {
4077    for r in start..end {
4078        // SAFETY: disjoint row ranges per worker.
4079        unsafe { *out.at(r) = q1_row_exact(bytes, r, gpr, x) };
4080    }
4081}
4082
4083/// q1t per-row overlay locator. After the base (`base_len`) come
4084/// `[u32 row_ptr[rows+1]]` then `[(u16 col, f16 val)]` grouped by row (row
4085/// `r`'s entries are `[row_ptr[r], row_ptr[r+1])`). Returns
4086/// `(row_ptr offset, entries offset, present)`.
4087fn q1t_overlay(bytes: &[u8], base_len: usize, rows: usize) -> (usize, usize, bool) {
4088    let entries = base_len + (rows + 1) * 4;
4089    (base_len, entries, entries <= bytes.len())
4090}
4091
4092/// Read `row_ptr[r]` from the overlay's prefix-sum table.
4093#[inline]
4094fn q1t_rowptr(bytes: &[u8], rp_off: usize, r: usize) -> usize {
4095    let o = rp_off + r * 4;
4096    u32::from_le_bytes([bytes[o], bytes[o + 1], bytes[o + 2], bytes[o + 3]]) as usize
4097}
4098
4099/// Byte → the 5 ternary signs it packs `{−1,0,+1}` as f32, precomputed so
4100/// decoding a q1t code is a table load, not the base-3 divide/modulo per
4101/// weight (division is ~20–40× the cost of a load). Built at compile time.
4102const SIGN5: [[f32; 5]; 256] = {
4103    let mut lut = [[0.0f32; 5]; 256];
4104    let pow3 = [1u16, 3, 9, 27, 81];
4105    let mut byte = 0usize;
4106    while byte < 256 {
4107        let mut i = 0usize;
4108        while i < 5 {
4109            let code = (byte as u16 / pow3[i]) % 3;
4110            lut[byte][i] = if code == 1 {
4111                1.0
4112            } else if code == 2 {
4113                -1.0
4114            } else {
4115                0.0
4116            };
4117            i += 1;
4118        }
4119        byte += 1;
4120    }
4121    lut
4122};
4123
4124/// Same table, as i8 signs — the operand for the int8 SDOT base kernel.
4125const SIGN5_I8: [[i8; 5]; 256] = {
4126    let mut lut = [[0i8; 5]; 256];
4127    let pow3 = [1u16, 3, 9, 27, 81];
4128    let mut byte = 0usize;
4129    while byte < 256 {
4130        let mut i = 0usize;
4131        while i < 5 {
4132            let code = (byte as u16 / pow3[i]) % 3;
4133            lut[byte][i] = if code == 1 {
4134                1
4135            } else if code == 2 {
4136                -1
4137            } else {
4138                0
4139            };
4140            i += 1;
4141        }
4142        byte += 1;
4143    }
4144    lut
4145};
4146
4147/// The same 5 i8 signs packed into a u64 (`[s0 s1 s2 s3 s4 0 0 0]`, LE) so the
4148/// group unpack is 7 unaligned u64 stores at offsets 0,5,10,…,30 instead of
4149/// six 5-byte copies + LUT indexing — each store's trailing zeros are fixed by
4150/// the next store, and the last one runs 6 B past the 32nd weight (the unpack
4151/// buffer is padded to 40). This is the decode/prefill hot inner op.
4152const SIGN5_U64: [u64; 256] = {
4153    let mut lut = [0u64; 256];
4154    let pow3 = [1u16, 3, 9, 27, 81];
4155    let mut byte = 0usize;
4156    while byte < 256 {
4157        let mut v = 0u64;
4158        let mut i = 0usize;
4159        while i < 5 {
4160            let code = (byte as u16 / pow3[i]) % 3;
4161            let s: u8 = if code == 1 {
4162                1
4163            } else if code == 2 {
4164                0xFF
4165            } else {
4166                0
4167            };
4168            v |= (s as u64) << (i * 8);
4169            i += 1;
4170        }
4171        lut[byte] = v;
4172        byte += 1;
4173    }
4174    lut
4175};
4176
4177/// Ternary base weight at `(row r, col j)` = `sign(code)·s_group`. Used to add
4178/// back activation-outlier columns, whose `x` was zeroed for the int8 bulk dot
4179/// (`split_act`). At a weight-outlier position the code is 0, so this is 0 and
4180/// the overlay correction owns that column — no double counting.
4181#[inline]
4182fn q1t_base_weight(bytes: &[u8], r: usize, gpr: usize, j: usize) -> f32 {
4183    const TILE: usize = cortiq_core::quant::Q1T_TILE;
4184    let off = (r * gpr + j / GROUP_SIZE) * TILE;
4185    let s = f16_to_f32(u16::from_le_bytes([bytes[off], bytes[off + 1]]));
4186    let within = j % GROUP_SIZE;
4187    SIGN5[bytes[off + 2 + within / 5] as usize][within % 5] * s
4188}
4189
4190/// One 32-group int8 dot via two SDOTs. Bit-exact vs the scalar i8 sum
4191/// (integer accumulation is order-independent).
4192#[cfg(target_arch = "aarch64")]
4193#[target_feature(enable = "neon,dotprod")]
4194#[inline]
4195unsafe fn sdot32_i8(w: *const i8, x: *const i8) -> i32 {
4196    // SAFETY: caller guarantees 32 readable i8 at each pointer.
4197    unsafe {
4198        use core::arch::aarch64::*;
4199        use core::arch::asm;
4200        let w0 = vld1q_s8(w);
4201        let w1 = vld1q_s8(w.add(16));
4202        let x0 = vld1q_s8(x);
4203        let x1 = vld1q_s8(x.add(16));
4204        let (mut a0, mut a1) = (vdupq_n_s32(0), vdupq_n_s32(0));
4205        asm!(
4206            "sdot {a0:v}.4s, {w0:v}.16b, {x0:v}.16b",
4207            "sdot {a1:v}.4s, {w1:v}.16b, {x1:v}.16b",
4208            a0 = inout(vreg) a0, a1 = inout(vreg) a1,
4209            w0 = in(vreg) w0, x0 = in(vreg) x0, w1 = in(vreg) w1, x1 = in(vreg) x1,
4210            options(pure, nomem, nostack),
4211        );
4212        vaddvq_s32(vaddq_s32(a0, a1))
4213    }
4214}
4215
4216/// One 32-group int8 dot via AVX2: signed·signed as `maddubs(|w|, sign(x,w))`
4217/// then `madd` and a horizontal reduce (the same idiom as `dot_q4t_row_avx2`).
4218#[cfg(target_arch = "x86_64")]
4219#[target_feature(enable = "avx2")]
4220#[inline]
4221unsafe fn i8dot32_avx2(w: *const i8, x: *const i8) -> i32 {
4222    // SAFETY: caller guarantees 32 readable i8 at each pointer.
4223    unsafe {
4224        use core::arch::x86_64::*;
4225        let wv = _mm256_loadu_si256(w as *const __m256i);
4226        let xv = _mm256_loadu_si256(x as *const __m256i);
4227        let p16 = _mm256_maddubs_epi16(_mm256_abs_epi8(wv), _mm256_sign_epi8(xv, wv));
4228        let d = _mm256_madd_epi16(p16, _mm256_set1_epi16(1));
4229        let hi128 = _mm256_extracti128_si256::<1>(d);
4230        let s128 = _mm_add_epi32(_mm256_castsi256_si128(d), hi128);
4231        let s64 = _mm_add_epi32(s128, _mm_srli_si128::<8>(s128));
4232        let s32 = _mm_add_epi32(s64, _mm_srli_si128::<4>(s64));
4233        _mm_cvtsi128_si32(s32)
4234    }
4235}
4236
4237/// Unpack one q1t group's base-3 codes into 32 i8 signs via 7 unaligned u64
4238/// stores (see `SIGN5_U64`). `dst` MUST have ≥ 40 bytes: the 7th store writes
4239/// `dst[30..38]`. Stores go in order so each one's trailing zeros are
4240/// overwritten by the next; the final 6 padding bytes are unused by the dot.
4241#[inline]
4242fn q1t_unpack_group_i8(codes: *const u8, dst: &mut [i8]) {
4243    debug_assert!(dst.len() >= 40);
4244    // SAFETY: codes points at 7 readable bytes; dst has ≥ 40 bytes so every
4245    // 8-byte store at offset bi*5 (bi ≤ 6 → ≤ 30) stays in bounds.
4246    unsafe {
4247        let p = dst.as_mut_ptr();
4248        for bi in 0..7 {
4249            core::ptr::write_unaligned(
4250                p.add(bi * 5) as *mut u64,
4251                SIGN5_U64[*codes.add(bi) as usize],
4252            );
4253        }
4254    }
4255}
4256
4257/// One 32-group int8 dot, arch-dispatched (the matmat inner loop, where the
4258/// row's signs are unpacked once and dotted against every batch input).
4259/// Callers are gated by `a8w8_enabled()`, so the target-feature arms are
4260/// reachable; the scalar arm is a non-SIMD-arch fallback.
4261#[inline]
4262fn q1t_i8dot32(w: *const i8, x: *const i8) -> i32 {
4263    #[cfg(target_arch = "aarch64")]
4264    unsafe {
4265        return sdot32_i8(w, x);
4266    }
4267    #[cfg(target_arch = "x86_64")]
4268    unsafe {
4269        return i8dot32_avx2(w, x);
4270    }
4271    #[allow(unreachable_code)]
4272    unsafe {
4273        let mut s = 0i32;
4274        for k in 0..GROUP_SIZE {
4275            s += *w.add(k) as i32 * *x.add(k) as i32;
4276        }
4277        s
4278    }
4279}
4280
4281#[inline]
4282unsafe fn q1t_unpack_reg_u64s(codes: *const u8) -> (u64, u64, u64, u64) {
4283    let (s0, s1, s2, s3, s4, s5, s6) = unsafe {
4284        (
4285            SIGN5_U64[*codes as usize],
4286            SIGN5_U64[*codes.add(1) as usize],
4287            SIGN5_U64[*codes.add(2) as usize],
4288            SIGN5_U64[*codes.add(3) as usize],
4289            SIGN5_U64[*codes.add(4) as usize],
4290            SIGN5_U64[*codes.add(5) as usize],
4291            SIGN5_U64[*codes.add(6) as usize],
4292        )
4293    };
4294
4295    let u0 = s0 | (s1 << 40);
4296    let u1 = (s1 >> 24) | (s2 << 16) | (s3 << 56);
4297    let u2 = (s3 >> 8) | (s4 << 32);
4298    let u3 = (s4 >> 32) | (s5 << 8) | (s6 << 48);
4299
4300    (u0, u1, u2, u3)
4301}
4302
4303/// One q1t row's int8 base dot: `Σ_group s·dot(signs, xq)` (before the shared
4304/// `sx`). Direct register unpacking (zero stack stores/loads, no STLF stalls).
4305/// ARM SDOT.
4306#[cfg(target_arch = "aarch64")]
4307#[target_feature(enable = "neon,dotprod")]
4308unsafe fn q1t_dot_row_sdot(bytes: &[u8], r: usize, gpr: usize, xq: &[i8]) -> f32 {
4309    use core::arch::aarch64::*;
4310    use core::arch::asm;
4311    unsafe {
4312        const TILE: usize = cortiq_core::quant::Q1T_TILE;
4313        let mut acc = 0f32;
4314        let bytes_ptr = bytes.as_ptr();
4315        let xq_ptr = xq.as_ptr();
4316        let row_off = r * gpr * TILE;
4317
4318        let gpr2 = gpr & !1;
4319        let mut gi = 0;
4320        while gi < gpr2 {
4321            let off0 = row_off + gi * TILE;
4322            let off1 = off0 + TILE;
4323            let s0 = f16_to_f32(u16::from_le_bytes([
4324                *bytes_ptr.add(off0),
4325                *bytes_ptr.add(off0 + 1),
4326            ]));
4327            let s1 = f16_to_f32(u16::from_le_bytes([
4328                *bytes_ptr.add(off1),
4329                *bytes_ptr.add(off1 + 1),
4330            ]));
4331
4332            let (u0_0, u1_0, u2_0, u3_0) = q1t_unpack_reg_u64s(bytes_ptr.add(off0 + 2));
4333            let (u0_1, u1_1, u2_1, u3_1) = q1t_unpack_reg_u64s(bytes_ptr.add(off1 + 2));
4334
4335            let w0_0 = vreinterpretq_s8_u64(vcombine_u64(vcreate_u64(u0_0), vcreate_u64(u1_0)));
4336            let w1_0 = vreinterpretq_s8_u64(vcombine_u64(vcreate_u64(u2_0), vcreate_u64(u3_0)));
4337            let w0_1 = vreinterpretq_s8_u64(vcombine_u64(vcreate_u64(u0_1), vcreate_u64(u1_1)));
4338            let w1_1 = vreinterpretq_s8_u64(vcombine_u64(vcreate_u64(u2_1), vcreate_u64(u3_1)));
4339
4340            let x0_0 = vld1q_s8(xq_ptr.add(gi * GROUP_SIZE));
4341            let x1_0 = vld1q_s8(xq_ptr.add(gi * GROUP_SIZE + 16));
4342            let x0_1 = vld1q_s8(xq_ptr.add((gi + 1) * GROUP_SIZE));
4343            let x1_1 = vld1q_s8(xq_ptr.add((gi + 1) * GROUP_SIZE + 16));
4344
4345            let (mut a0_0, mut a1_0) = (vdupq_n_s32(0), vdupq_n_s32(0));
4346            let (mut a0_1, mut a1_1) = (vdupq_n_s32(0), vdupq_n_s32(0));
4347            asm!(
4348                "sdot {a0_0:v}.4s, {w0_0:v}.16b, {x0_0:v}.16b",
4349                "sdot {a1_0:v}.4s, {w1_0:v}.16b, {x1_0:v}.16b",
4350                "sdot {a0_1:v}.4s, {w0_1:v}.16b, {x0_1:v}.16b",
4351                "sdot {a1_1:v}.4s, {w1_1:v}.16b, {x1_1:v}.16b",
4352                a0_0 = inout(vreg) a0_0, a1_0 = inout(vreg) a1_0,
4353                a0_1 = inout(vreg) a0_1, a1_1 = inout(vreg) a1_1,
4354                w0_0 = in(vreg) w0_0, x0_0 = in(vreg) x0_0, w1_0 = in(vreg) w1_0, x1_0 = in(vreg) x1_0,
4355                w0_1 = in(vreg) w0_1, x0_1 = in(vreg) x0_1, w1_1 = in(vreg) w1_1, x1_1 = in(vreg) x1_1,
4356                options(pure, nomem, nostack),
4357            );
4358            let d0 = vaddvq_s32(vaddq_s32(a0_0, a1_0));
4359            let d1 = vaddvq_s32(vaddq_s32(a0_1, a1_1));
4360            acc += d0 as f32 * s0 + d1 as f32 * s1;
4361            gi += 2;
4362        }
4363
4364        if gi < gpr {
4365            let off = row_off + gi * TILE;
4366            let s = f16_to_f32(u16::from_le_bytes([
4367                *bytes_ptr.add(off),
4368                *bytes_ptr.add(off + 1),
4369            ]));
4370            let (u0, u1, u2, u3) = q1t_unpack_reg_u64s(bytes_ptr.add(off + 2));
4371            let w0 = vreinterpretq_s8_u64(vcombine_u64(vcreate_u64(u0), vcreate_u64(u1)));
4372            let w1 = vreinterpretq_s8_u64(vcombine_u64(vcreate_u64(u2), vcreate_u64(u3)));
4373            let x0 = vld1q_s8(xq_ptr.add(gi * GROUP_SIZE));
4374            let x1 = vld1q_s8(xq_ptr.add(gi * GROUP_SIZE + 16));
4375            let (mut a0, mut a1) = (vdupq_n_s32(0), vdupq_n_s32(0));
4376            asm!(
4377                "sdot {a0:v}.4s, {w0:v}.16b, {x0:v}.16b",
4378                "sdot {a1:v}.4s, {w1:v}.16b, {x1:v}.16b",
4379                a0 = inout(vreg) a0, a1 = inout(vreg) a1,
4380                w0 = in(vreg) w0, x0 = in(vreg) x0, w1 = in(vreg) w1, x1 = in(vreg) x1,
4381                options(pure, nomem, nostack),
4382            );
4383            let d = vaddvq_s32(vaddq_s32(a0, a1));
4384            acc += d as f32 * s;
4385        }
4386        acc
4387    }
4388}
4389
4390/// x86 AVX2 mirror of `q1t_dot_row_sdot` (maddubs int8 dot per group).
4391#[cfg(target_arch = "x86_64")]
4392#[target_feature(enable = "avx2")]
4393unsafe fn q1t_dot_row_avx2(bytes: &[u8], r: usize, gpr: usize, xq: &[i8]) -> f32 {
4394    use core::arch::x86_64::*;
4395    unsafe {
4396        const TILE: usize = cortiq_core::quant::Q1T_TILE;
4397        let mut acc = 0f32;
4398        let bytes_ptr = bytes.as_ptr();
4399        let xq_ptr = xq.as_ptr();
4400        let row_off = r * gpr * TILE;
4401
4402        let ones = _mm256_set1_epi16(1);
4403        for gi in 0..gpr {
4404            let off = row_off + gi * TILE;
4405            let s = f16_to_f32(u16::from_le_bytes([
4406                *bytes_ptr.add(off),
4407                *bytes_ptr.add(off + 1),
4408            ]));
4409            let (u0, u1, u2, u3) = q1t_unpack_reg_u64s(bytes_ptr.add(off + 2));
4410            let wv = _mm256_set_epi64x(u3 as i64, u2 as i64, u1 as i64, u0 as i64);
4411            let xv = _mm256_loadu_si256(xq_ptr.add(gi * GROUP_SIZE) as *const __m256i);
4412            let p16 = _mm256_maddubs_epi16(_mm256_abs_epi8(wv), _mm256_sign_epi8(xv, wv));
4413            let d256 = _mm256_madd_epi16(p16, ones);
4414            let d128 = _mm_add_epi32(
4415                _mm256_castsi256_si128(d256),
4416                _mm256_extracti128_si256(d256, 1),
4417            );
4418            let d64 = _mm_add_epi32(d128, _mm_shuffle_epi32(d128, 0xee));
4419            let d32 = _mm_cvtsi128_si32(_mm_add_epi32(d64, _mm_shuffle_epi32(d64, 0x55)));
4420            acc += d32 as f32 * s;
4421        }
4422        acc
4423    }
4424}
4425
4426/// VNNI twin of `q1t_dot_row_avx2` (see `dpbusd_hsum`).
4427#[cfg(target_arch = "x86_64")]
4428#[target_feature(enable = "avx2,avx512f,avx512bw,avx512vl,avx512vnni")]
4429unsafe fn q1t_dot_row_vnni(bytes: &[u8], r: usize, gpr: usize, xq: &[i8]) -> f32 {
4430    use core::arch::x86_64::*;
4431    // SAFETY: same tile/xq contracts as `q1t_dot_row_avx2`.
4432    unsafe {
4433        const TILE: usize = cortiq_core::quant::Q1T_TILE;
4434        let mut acc = 0f32;
4435        let bytes_ptr = bytes.as_ptr();
4436        let xq_ptr = xq.as_ptr();
4437        let row_off = r * gpr * TILE;
4438        for gi in 0..gpr {
4439            let off = row_off + gi * TILE;
4440            let s = f16_to_f32(u16::from_le_bytes([
4441                *bytes_ptr.add(off),
4442                *bytes_ptr.add(off + 1),
4443            ]));
4444            let (u0, u1, u2, u3) = q1t_unpack_reg_u64s(bytes_ptr.add(off + 2));
4445            let wv = _mm256_set_epi64x(u3 as i64, u2 as i64, u1 as i64, u0 as i64);
4446            let xv = _mm256_loadu_si256(xq_ptr.add(gi * GROUP_SIZE) as *const __m256i);
4447            let d = dpbusd_hsum(_mm256_abs_epi8(wv), _mm256_sign_epi8(xv, wv));
4448            acc += d as f32 * s;
4449        }
4450        acc
4451    }
4452}
4453
4454/// Per-row int8 base dot, dispatched once per row (matvec decode hot path).
4455/// Callers are gated by `a8w8_enabled()`, so the target-feature kernels are
4456/// reachable.
4457#[inline]
4458fn q1t_dot_row_i8(bytes: &[u8], r: usize, gpr: usize, xq: &[i8]) -> f32 {
4459    #[cfg(target_arch = "aarch64")]
4460    unsafe {
4461        return q1t_dot_row_sdot(bytes, r, gpr, xq);
4462    }
4463    #[cfg(target_arch = "x86_64")]
4464    unsafe {
4465        if vnni_tiles_enabled() {
4466            return q1t_dot_row_vnni(bytes, r, gpr, xq);
4467        }
4468        return q1t_dot_row_avx2(bytes, r, gpr, xq);
4469    }
4470    #[allow(unreachable_code)]
4471    {
4472        const TILE: usize = cortiq_core::quant::Q1T_TILE;
4473        let mut acc = 0f32;
4474        let mut sg = [0i8; GROUP_SIZE + 8]; // +8 slack for the u64-store unpack
4475        for gi in 0..gpr {
4476            let off = (r * gpr + gi) * TILE;
4477            let s = f16_to_f32(u16::from_le_bytes([bytes[off], bytes[off + 1]]));
4478            q1t_unpack_group_i8(bytes.as_ptr().wrapping_add(off + 2), &mut sg);
4479            let mut d = 0i32;
4480            for k in 0..GROUP_SIZE {
4481                d += sg[k] as i32 * xq[gi * GROUP_SIZE + k] as i32;
4482            }
4483            acc += d as f32 * s;
4484        }
4485        acc
4486    }
4487}
4488
4489/// Σ over a row's outliers of `value·x[col]` — the correction that adds the
4490/// overlay's exact weights on top of the base dot. INVARIANT: the encoder
4491/// writes ternary code 0 at every outlier position (`quantize_q1t`), so the
4492/// base contributes nothing there and this is a plain `value·x`, not
4493/// `(value − base)·x` — no scattered per-outlier scale read. Row `r`'s entries
4494/// are the contiguous slice `[row_ptr[r], row_ptr[r+1])`, so no binary search.
4495fn q1t_row_outlier_correction(
4496    bytes: &[u8],
4497    r: usize,
4498    rp_off: usize,
4499    entries_off: usize,
4500    has_ov: bool,
4501    x: &[f32],
4502) -> f32 {
4503    if !has_ov {
4504        return 0.0;
4505    }
4506    let (c0, c1) = (
4507        q1t_rowptr(bytes, rp_off, r),
4508        q1t_rowptr(bytes, rp_off, r + 1),
4509    );
4510    let mut corr = 0f32;
4511    for p in c0..c1 {
4512        let e = entries_off + p * 4;
4513        let col = u16::from_le_bytes([bytes[e], bytes[e + 1]]) as usize;
4514        let val = f16_to_f32(u16::from_le_bytes([bytes[e + 2], bytes[e + 3]]));
4515        corr += val * x[col];
4516    }
4517    corr
4518}
4519
4520/// Dequantize one q1t row into `buf[..cols]` via the sign LUT (no division),
4521/// then apply the row's outliers (its `[row_ptr[r], row_ptr[r+1])` slice).
4522/// Used by the batched (prefill) path where the decode amortizes over the batch.
4523fn q1t_dequant_row(
4524    bytes: &[u8],
4525    r: usize,
4526    gpr: usize,
4527    rp_off: usize,
4528    entries_off: usize,
4529    has_ov: bool,
4530    buf: &mut [f32],
4531) {
4532    const TILE: usize = cortiq_core::quant::Q1T_TILE;
4533    for g in 0..gpr {
4534        let off = (r * gpr + g) * TILE;
4535        let s = f16_to_f32(u16::from_le_bytes([bytes[off], bytes[off + 1]]));
4536        let codes = &bytes[off + 2..off + TILE];
4537        let bc = g * GROUP_SIZE;
4538        // 6 full bytes (30 codes) + a 7th byte holding the last 2.
4539        for bi in 0..6 {
4540            let lut = &SIGN5[codes[bi] as usize];
4541            let d = &mut buf[bc + bi * 5..bc + bi * 5 + 5];
4542            for i in 0..5 {
4543                d[i] = lut[i] * s;
4544            }
4545        }
4546        let lut = &SIGN5[codes[6] as usize];
4547        buf[bc + 30] = lut[0] * s;
4548        buf[bc + 31] = lut[1] * s;
4549    }
4550    if !has_ov {
4551        return;
4552    }
4553    let (c0, c1) = (
4554        q1t_rowptr(bytes, rp_off, r),
4555        q1t_rowptr(bytes, rp_off, r + 1),
4556    );
4557    for p in c0..c1 {
4558        let e = entries_off + p * 4;
4559        let col = u16::from_le_bytes([bytes[e], bytes[e + 1]]) as usize;
4560        buf[col] = f16_to_f32(u16::from_le_bytes([bytes[e + 2], bytes[e + 3]]));
4561    }
4562}
4563
4564/// Add the sparse outlier overlay onto a base dot already in `out` (the GPU
4565/// computes the ternary base; the overlay stays on the CPU — its entries are
4566/// few and its per-row gather doesn't vectorize on the GPU). Row-parallel.
4567fn q1t_add_overlay(
4568    bytes: &[u8],
4569    x: &[f32],
4570    rows: usize,
4571    cols: usize,
4572    out: &mut [f32],
4573    pool: Option<&Pool>,
4574) {
4575    const TILE: usize = cortiq_core::quant::Q1T_TILE;
4576    let gpr = cols / GROUP_SIZE;
4577    let (rp_off, ent_off, has_ov) = q1t_overlay(bytes, rows * gpr * TILE, rows);
4578    if !has_ov {
4579        return;
4580    }
4581    let out_addr = SendMut(out.as_mut_ptr());
4582    let run = move |start: usize, end: usize| {
4583        for r in start..end {
4584            let corr = q1t_row_outlier_correction(bytes, r, rp_off, ent_off, has_ov, x);
4585            // SAFETY: disjoint rows; add onto the base the GPU already wrote.
4586            unsafe { *out_addr.at(r) += corr };
4587        }
4588    };
4589    dispatch_rows(pool, rows, &run);
4590}
4591
4592/// Q1T row range via the A8W8 int8 path — shared activation split,
4593/// per-row: base SDOT dot + outlier correction + overlay.
4594#[allow(clippy::too_many_arguments)]
4595fn q1t_range_a8w8(
4596    bytes: &[u8],
4597    gpr: usize,
4598    rp_off: usize,
4599    ent_off: usize,
4600    has_ov: bool,
4601    act: &SplitAct,
4602    x: &[f32],
4603    out: SendMut,
4604    start: usize,
4605    end: usize,
4606) {
4607    for r in start..end {
4608        let mut acc = q1t_dot_row_i8(bytes, r, gpr, &act.xq) * act.sx;
4609        for &(j, xv) in &act.outliers {
4610            acc += q1t_base_weight(bytes, r, gpr, j) * xv;
4611        }
4612        acc += q1t_row_outlier_correction(bytes, r, rp_off, ent_off, has_ov, x);
4613        // SAFETY: disjoint row ranges per worker.
4614        unsafe { *out.at(r) = acc };
4615    }
4616}
4617
4618/// Q1T row range via the f32 path (no SDOT) — for matvec_many batched
4619/// dispatch when a8w8 is unavailable.
4620#[allow(clippy::too_many_arguments)]
4621fn q1t_range_f32_batch(
4622    bytes: &[u8],
4623    gpr: usize,
4624    rp_off: usize,
4625    ent_off: usize,
4626    has_ov: bool,
4627    x: &[f32],
4628    out: SendMut,
4629    start: usize,
4630    end: usize,
4631) {
4632    const TILE: usize = cortiq_core::quant::Q1T_TILE;
4633    let mut sg = [0f32; GROUP_SIZE];
4634    for r in start..end {
4635        let mut acc = 0f32;
4636        for g in 0..gpr {
4637            let off = (r * gpr + g) * TILE;
4638            let s = f16_to_f32(u16::from_le_bytes([bytes[off], bytes[off + 1]]));
4639            let codes = &bytes[off + 2..off + TILE];
4640            let xg = &x[g * GROUP_SIZE..g * GROUP_SIZE + GROUP_SIZE];
4641            for bi in 0..6 {
4642                sg[bi * 5..bi * 5 + 5].copy_from_slice(&SIGN5[codes[bi] as usize]);
4643            }
4644            let lut = &SIGN5[codes[6] as usize];
4645            sg[30] = lut[0];
4646            sg[31] = lut[1];
4647            let mut gsum = 0f32;
4648            for k in 0..GROUP_SIZE {
4649                gsum += sg[k] * xg[k];
4650            }
4651            acc += s * gsum;
4652        }
4653        acc += q1t_row_outlier_correction(bytes, r, rp_off, ent_off, has_ov, x);
4654        // SAFETY: disjoint row ranges per worker.
4655        unsafe { *out.at(r) = acc };
4656    }
4657}
4658
4659/// Ternary (q1t) matvec — decode+dot straight from mmap, one group at a time:
4660/// no per-ROW buffer, no division (the sign LUT), and a tiny per-group sign
4661/// buffer so the 32-wide dot vectorizes. This is the decode hot path.
4662fn q1t_matvec(
4663    bytes: &[u8],
4664    x: &[f32],
4665    rows: usize,
4666    cols: usize,
4667    out: &mut [f32],
4668    pool: Option<&Pool>,
4669) {
4670    debug_assert_eq!(out.len(), rows);
4671    const TILE: usize = cortiq_core::quant::Q1T_TILE;
4672    let gpr = cols / GROUP_SIZE;
4673    let (rp_off, ent_off, has_ov) = q1t_overlay(bytes, rows * gpr * TILE, rows);
4674    let out_addr = SendMut(out.as_mut_ptr());
4675    // int8 SDOT base dot (ARM dotprod): ~4× the f32 arithmetic. x → i8 once
4676    // (`split_act`), activation outliers added back exactly in f32, weight
4677    // overlay on top. ARM SDOT / x86 AVX2; CMF_SDOT=0 keeps the exact f32 path.
4678    if a8w8_enabled() {
4679        let act = split_act(x);
4680        let act = &act;
4681        let run = move |start: usize, end: usize| {
4682            for r in start..end {
4683                let mut acc = q1t_dot_row_i8(bytes, r, gpr, &act.xq) * act.sx;
4684                for &(j, xv) in &act.outliers {
4685                    acc += q1t_base_weight(bytes, r, gpr, j) * xv;
4686                }
4687                acc += q1t_row_outlier_correction(bytes, r, rp_off, ent_off, has_ov, x);
4688                // SAFETY: disjoint row ranges per worker.
4689                unsafe { *out_addr.at(r) = acc };
4690            }
4691        };
4692        dispatch_rows(pool, rows, &run);
4693        return;
4694    }
4695    let run = move |start: usize, end: usize| {
4696        // Per-group signs, unpacked contiguously so the dot below is a clean
4697        // 32-wide reduction the autovectorizer turns into f32x4 FMAs — the
4698        // 5-values-per-byte base-3 layout won't SIMD in place.
4699        let mut sg = [0f32; GROUP_SIZE];
4700        for r in start..end {
4701            let mut acc = 0f32;
4702            for g in 0..gpr {
4703                let off = (r * gpr + g) * TILE;
4704                let s = f16_to_f32(u16::from_le_bytes([bytes[off], bytes[off + 1]]));
4705                let codes = &bytes[off + 2..off + TILE];
4706                let xg = &x[g * GROUP_SIZE..g * GROUP_SIZE + GROUP_SIZE];
4707                for bi in 0..6 {
4708                    sg[bi * 5..bi * 5 + 5].copy_from_slice(&SIGN5[codes[bi] as usize]);
4709                }
4710                let lut = &SIGN5[codes[6] as usize];
4711                sg[30] = lut[0];
4712                sg[31] = lut[1];
4713                let mut gsum = 0f32;
4714                for k in 0..GROUP_SIZE {
4715                    gsum += sg[k] * xg[k];
4716                }
4717                acc += s * gsum;
4718            }
4719            acc += q1t_row_outlier_correction(bytes, r, rp_off, ent_off, has_ov, x);
4720            unsafe { *out_addr.at(r) = acc };
4721        }
4722    };
4723    dispatch_rows(pool, rows, &run);
4724}
4725
4726/// Fused-pair twin of `q1t_dot_row_sdot`: ONE register unpack of the
4727/// ternary codes serves BOTH activation streams (the unpack chain is
4728/// the dominant per-row cost — MTP verify pairs paid it twice). Per
4729/// stream the group order and f32 accumulation match the single-row
4730/// kernel exactly, so pair == 2×matvec bit-for-bit.
4731#[cfg(target_arch = "aarch64")]
4732#[target_feature(enable = "neon,dotprod")]
4733unsafe fn q1t_dot_row_sdot2(bytes: &[u8], r: usize, gpr: usize, xa: &[i8], xb: &[i8]) -> [f32; 2] {
4734    use core::arch::aarch64::*;
4735    use core::arch::asm;
4736    // SAFETY: same slice-length contracts as `q1t_dot_row_sdot`, ×2.
4737    unsafe {
4738        const TILE: usize = cortiq_core::quant::Q1T_TILE;
4739        let bytes_ptr = bytes.as_ptr();
4740        let row_off = r * gpr * TILE;
4741        let xp = [xa.as_ptr(), xb.as_ptr()];
4742        let mut acc = [0f32; 2];
4743        macro_rules! sdot2 {
4744            ($w0:expr, $w1:expr, $x:expr) => {{
4745                let x0 = vld1q_s8($x);
4746                let x1 = vld1q_s8($x.add(16));
4747                let (mut a0, mut a1) = (vdupq_n_s32(0), vdupq_n_s32(0));
4748                asm!(
4749                    "sdot {a0:v}.4s, {w0:v}.16b, {x0:v}.16b",
4750                    "sdot {a1:v}.4s, {w1:v}.16b, {x1:v}.16b",
4751                    a0 = inout(vreg) a0, a1 = inout(vreg) a1,
4752                    w0 = in(vreg) $w0, x0 = in(vreg) x0, w1 = in(vreg) $w1, x1 = in(vreg) x1,
4753                    options(pure, nomem, nostack),
4754                );
4755                vaddvq_s32(vaddq_s32(a0, a1))
4756            }};
4757        }
4758        let gpr2 = gpr & !1;
4759        let mut gi = 0;
4760        while gi < gpr2 {
4761            let off0 = row_off + gi * TILE;
4762            let off1 = off0 + TILE;
4763            let s0 = f16_to_f32(u16::from_le_bytes([
4764                *bytes_ptr.add(off0),
4765                *bytes_ptr.add(off0 + 1),
4766            ]));
4767            let s1 = f16_to_f32(u16::from_le_bytes([
4768                *bytes_ptr.add(off1),
4769                *bytes_ptr.add(off1 + 1),
4770            ]));
4771            let (u0_0, u1_0, u2_0, u3_0) = q1t_unpack_reg_u64s(bytes_ptr.add(off0 + 2));
4772            let (u0_1, u1_1, u2_1, u3_1) = q1t_unpack_reg_u64s(bytes_ptr.add(off1 + 2));
4773            let w0_0 = vreinterpretq_s8_u64(vcombine_u64(vcreate_u64(u0_0), vcreate_u64(u1_0)));
4774            let w1_0 = vreinterpretq_s8_u64(vcombine_u64(vcreate_u64(u2_0), vcreate_u64(u3_0)));
4775            let w0_1 = vreinterpretq_s8_u64(vcombine_u64(vcreate_u64(u0_1), vcreate_u64(u1_1)));
4776            let w1_1 = vreinterpretq_s8_u64(vcombine_u64(vcreate_u64(u2_1), vcreate_u64(u3_1)));
4777            for k in 0..2 {
4778                let d0 = sdot2!(w0_0, w1_0, xp[k].add(gi * GROUP_SIZE));
4779                let d1 = sdot2!(w0_1, w1_1, xp[k].add((gi + 1) * GROUP_SIZE));
4780                acc[k] += d0 as f32 * s0 + d1 as f32 * s1;
4781            }
4782            gi += 2;
4783        }
4784        if gi < gpr {
4785            let off = row_off + gi * TILE;
4786            let s = f16_to_f32(u16::from_le_bytes([
4787                *bytes_ptr.add(off),
4788                *bytes_ptr.add(off + 1),
4789            ]));
4790            let (u0, u1, u2, u3) = q1t_unpack_reg_u64s(bytes_ptr.add(off + 2));
4791            let w0 = vreinterpretq_s8_u64(vcombine_u64(vcreate_u64(u0), vcreate_u64(u1)));
4792            let w1 = vreinterpretq_s8_u64(vcombine_u64(vcreate_u64(u2), vcreate_u64(u3)));
4793            for k in 0..2 {
4794                let d = sdot2!(w0, w1, xp[k].add(gi * GROUP_SIZE));
4795                acc[k] += d as f32 * s;
4796            }
4797        }
4798        acc
4799    }
4800}
4801
4802/// Fused Q1T pair matvec: ONE pass over the rows serves both
4803/// activation streams — on ARM the ternary register unpack happens
4804/// once per tile pair (`q1t_dot_row_sdot2`); elsewhere the second dot
4805/// rides the row's L1-warm tile bytes. Per stream the math matches
4806/// `q1t_matvec` exactly.
4807fn q1t_matvec2(
4808    bytes: &[u8],
4809    x1: &[f32],
4810    x2: &[f32],
4811    rows: usize,
4812    cols: usize,
4813    o1: &mut [f32],
4814    o2: &mut [f32],
4815    pool: Option<&Pool>,
4816) {
4817    debug_assert_eq!(o1.len(), rows);
4818    debug_assert_eq!(o2.len(), rows);
4819    const TILE: usize = cortiq_core::quant::Q1T_TILE;
4820    let gpr = cols / GROUP_SIZE;
4821    let (rp_off, ent_off, has_ov) = q1t_overlay(bytes, rows * gpr * TILE, rows);
4822    let out1 = SendMut(o1.as_mut_ptr());
4823    let out2 = SendMut(o2.as_mut_ptr());
4824    if a8w8_enabled() {
4825        let a1 = split_act(x1);
4826        let a2 = split_act(x2);
4827        let (a1, a2) = (&a1, &a2);
4828        let run = move |start: usize, end: usize| {
4829            for r in start..end {
4830                #[cfg(target_arch = "aarch64")]
4831                // a8w8 on aarch64 ⇔ sdot_enabled(), so the kernel's
4832                // target features are present.
4833                let ds = unsafe { q1t_dot_row_sdot2(bytes, r, gpr, &a1.xq, &a2.xq) };
4834                #[cfg(not(target_arch = "aarch64"))]
4835                let ds = [
4836                    q1t_dot_row_i8(bytes, r, gpr, &a1.xq),
4837                    q1t_dot_row_i8(bytes, r, gpr, &a2.xq),
4838                ];
4839                let mut acc1 = ds[0] * a1.sx;
4840                for &(j, xv) in &a1.outliers {
4841                    acc1 += q1t_base_weight(bytes, r, gpr, j) * xv;
4842                }
4843                acc1 += q1t_row_outlier_correction(bytes, r, rp_off, ent_off, has_ov, x1);
4844                let mut acc2 = ds[1] * a2.sx;
4845                for &(j, xv) in &a2.outliers {
4846                    acc2 += q1t_base_weight(bytes, r, gpr, j) * xv;
4847                }
4848                acc2 += q1t_row_outlier_correction(bytes, r, rp_off, ent_off, has_ov, x2);
4849                // SAFETY: disjoint row ranges per worker.
4850                unsafe {
4851                    *out1.at(r) = acc1;
4852                    *out2.at(r) = acc2;
4853                }
4854            }
4855        };
4856        dispatch_rows(pool, rows, &run);
4857        return;
4858    }
4859    let run = move |start: usize, end: usize| {
4860        // Exact path (CMF_SDOT=0): unpack the sign LUT once per group,
4861        // dot both streams — same op order per stream as `q1t_matvec`.
4862        let mut sg = [0f32; GROUP_SIZE];
4863        for r in start..end {
4864            let mut acc1 = 0f32;
4865            let mut acc2 = 0f32;
4866            for g in 0..gpr {
4867                let off = (r * gpr + g) * TILE;
4868                let s = f16_to_f32(u16::from_le_bytes([bytes[off], bytes[off + 1]]));
4869                let codes = &bytes[off + 2..off + TILE];
4870                for bi in 0..6 {
4871                    sg[bi * 5..bi * 5 + 5].copy_from_slice(&SIGN5[codes[bi] as usize]);
4872                }
4873                let lut = &SIGN5[codes[6] as usize];
4874                sg[30] = lut[0];
4875                sg[31] = lut[1];
4876                let xg1 = &x1[g * GROUP_SIZE..g * GROUP_SIZE + GROUP_SIZE];
4877                let xg2 = &x2[g * GROUP_SIZE..g * GROUP_SIZE + GROUP_SIZE];
4878                let mut gsum1 = 0f32;
4879                for k in 0..GROUP_SIZE {
4880                    gsum1 += sg[k] * xg1[k];
4881                }
4882                acc1 += s * gsum1;
4883                let mut gsum2 = 0f32;
4884                for k in 0..GROUP_SIZE {
4885                    gsum2 += sg[k] * xg2[k];
4886                }
4887                acc2 += s * gsum2;
4888            }
4889            acc1 += q1t_row_outlier_correction(bytes, r, rp_off, ent_off, has_ov, x1);
4890            acc2 += q1t_row_outlier_correction(bytes, r, rp_off, ent_off, has_ov, x2);
4891            // SAFETY: disjoint row ranges per worker.
4892            unsafe {
4893                *out1.at(r) = acc1;
4894                *out2.at(r) = acc2;
4895            }
4896        }
4897    };
4898    dispatch_rows(pool, rows, &run);
4899}
4900
4901/// Ternary (q1t) matmat (prefill) — dequant each row once, dot the whole
4902/// batch against it (amortizes the per-row decode).
4903fn q1t_matmat(
4904    bytes: &[u8],
4905    xs: &[f32],
4906    b: usize,
4907    rows: usize,
4908    cols: usize,
4909    out: &mut [f32],
4910    pool: Option<&Pool>,
4911) {
4912    debug_assert_eq!(out.len(), b * rows);
4913    const TILE: usize = cortiq_core::quant::Q1T_TILE;
4914    let gpr = cols / GROUP_SIZE;
4915    let (rp_off, ent_off, has_ov) = q1t_overlay(bytes, rows * gpr * TILE, rows);
4916    let out_addr = SendMut(out.as_mut_ptr());
4917    // int8 prefill (ARM SDOT / x86 AVX2): quantize the B inputs once, unpack
4918    // each weight row's signs to i8 ONCE, then int8-dot against every input —
4919    // the row sign-decode amortizes over the whole batch. CMF_SDOT=0 → f32.
4920    if a8w8_enabled() {
4921        let acts: Vec<SplitAct> = (0..b)
4922            .map(|bi| split_act(&xs[bi * cols..(bi + 1) * cols]))
4923            .collect();
4924        let acts = &acts;
4925        let run = move |start: usize, end: usize| {
4926            let mut sg = vec![0i8; cols + 8]; // row signs, i8 (+8 unpack slack)
4927            let mut sc = vec![0f32; gpr]; // per-group scales
4928            let mut accs = vec![0f32; b]; // per-batch accumulators, reused per row
4929            for r in start..end {
4930                for g in 0..gpr {
4931                    let off = (r * gpr + g) * TILE;
4932                    sc[g] = f16_to_f32(u16::from_le_bytes([bytes[off], bytes[off + 1]]));
4933                    q1t_unpack_group_i8(
4934                        bytes.as_ptr().wrapping_add(off + 2),
4935                        &mut sg[g * GROUP_SIZE..],
4936                    );
4937                }
4938                for bi in 0..b {
4939                    let act = &acts[bi];
4940                    let mut isum = 0f32;
4941                    for g in 0..gpr {
4942                        let d = q1t_i8dot32(
4943                            sg.as_ptr().wrapping_add(g * GROUP_SIZE),
4944                            act.xq.as_ptr().wrapping_add(g * GROUP_SIZE),
4945                        );
4946                        isum += d as f32 * sc[g];
4947                    }
4948                    let mut acc = isum * act.sx;
4949                    for &(j, xv) in &act.outliers {
4950                        acc += q1t_base_weight(bytes, r, gpr, j) * xv;
4951                    }
4952                    accs[bi] = acc;
4953                }
4954                // Overlay ONCE per row for the whole batch: read each (col, val)
4955                // from mmap a single time (was b× — the re-read dominated prefill)
4956                // and fan it out over the batch via the cached inputs.
4957                if has_ov {
4958                    let (c0, c1) = (
4959                        q1t_rowptr(bytes, rp_off, r),
4960                        q1t_rowptr(bytes, rp_off, r + 1),
4961                    );
4962                    for p in c0..c1 {
4963                        let e = ent_off + p * 4;
4964                        let col = u16::from_le_bytes([bytes[e], bytes[e + 1]]) as usize;
4965                        let val = f16_to_f32(u16::from_le_bytes([bytes[e + 2], bytes[e + 3]]));
4966                        for bi in 0..b {
4967                            accs[bi] += val * xs[bi * cols + col];
4968                        }
4969                    }
4970                }
4971                for bi in 0..b {
4972                    unsafe { *out_addr.at(bi * rows + r) = accs[bi] };
4973                }
4974            }
4975        };
4976        dispatch_rows(pool, rows, &run);
4977        return;
4978    }
4979    let run = move |start: usize, end: usize| {
4980        let mut buf = vec![0f32; cols];
4981        for r in start..end {
4982            q1t_dequant_row(bytes, r, gpr, rp_off, ent_off, has_ov, &mut buf);
4983            for bi in 0..b {
4984                let xr = &xs[bi * cols..(bi + 1) * cols];
4985                let mut acc = 0f32;
4986                for j in 0..cols {
4987                    acc += buf[j] * xr[j];
4988                }
4989                unsafe { *out_addr.at(bi * rows + r) = acc };
4990            }
4991        }
4992    };
4993    dispatch_rows(pool, rows, &run);
4994}
4995
4996fn q1_matvec(
4997    bytes: &[u8],
4998    x: &[f32],
4999    rows: usize,
5000    cols: usize,
5001    out: &mut [f32],
5002    pool: Option<&Pool>,
5003) {
5004    debug_assert_eq!(out.len(), rows);
5005    let gpr = cols / GROUP_SIZE;
5006    let out_addr = SendMut(out.as_mut_ptr());
5007    if a8w8_enabled() {
5008        let act = split_act(x);
5009        let gsum = q1_group_sums(&act.xq, gpr);
5010        let (act, gsum) = (&act, &gsum);
5011        let run = move |start: usize, end: usize| {
5012            q1_range_a8w8(bytes, gpr, act, gsum, out_addr, start, end)
5013        };
5014        dispatch_rows(pool, rows, &run);
5015        return;
5016    }
5017    let run = move |start: usize, end: usize| q1_range_f32(bytes, gpr, x, out_addr, start, end);
5018    dispatch_rows(pool, rows, &run);
5019}
5020
5021/// Fused two-input q1 matvec (weights read once per pair).
5022#[allow(clippy::too_many_arguments)]
5023fn q1_matvec2(
5024    bytes: &[u8],
5025    x1: &[f32],
5026    x2: &[f32],
5027    rows: usize,
5028    cols: usize,
5029    o1: &mut [f32],
5030    o2: &mut [f32],
5031    pool: Option<&Pool>,
5032) {
5033    let gpr = cols / GROUP_SIZE;
5034    let p1 = SendMut(o1.as_mut_ptr());
5035    let p2 = SendMut(o2.as_mut_ptr());
5036    if a8w8_enabled() {
5037        let a1 = split_act(x1);
5038        let a2 = split_act(x2);
5039        let g1 = q1_group_sums(&a1.xq, gpr);
5040        let g2 = q1_group_sums(&a2.xq, gpr);
5041        let (a1, a2, g1, g2) = (&a1, &a2, &g1, &g2);
5042        let run = move |start: usize, end: usize| {
5043            for r in start..end {
5044                let mut v1 = dot_q1_row_i8(bytes, r, gpr, &a1.xq, g1) * a1.sx;
5045                let mut v2 = dot_q1_row_i8(bytes, r, gpr, &a2.xq, g2) * a2.sx;
5046                for &(j, xv) in &a1.outliers {
5047                    let (w, s) = q1_outlier(bytes, r, gpr, j);
5048                    v1 += w * s * xv;
5049                }
5050                for &(j, xv) in &a2.outliers {
5051                    let (w, s) = q1_outlier(bytes, r, gpr, j);
5052                    v2 += w * s * xv;
5053                }
5054                // SAFETY: disjoint row ranges per worker.
5055                unsafe {
5056                    *p1.at(r) = v1;
5057                    *p2.at(r) = v2;
5058                }
5059            }
5060        };
5061        dispatch_rows(pool, rows, &run);
5062        return;
5063    }
5064    let run = move |start: usize, end: usize| {
5065        for r in start..end {
5066            // SAFETY: disjoint row ranges per worker.
5067            unsafe {
5068                *p1.at(r) = q1_row_exact(bytes, r, gpr, x1);
5069                *p2.at(r) = q1_row_exact(bytes, r, gpr, x2);
5070            }
5071        }
5072    };
5073    dispatch_rows(pool, rows, &run);
5074}
5075
5076/// Batched q1 matmat: each row's tiles stream once per microbatch.
5077#[allow(clippy::too_many_arguments)]
5078fn q1_matmat(
5079    bytes: &[u8],
5080    xs_all: &[f32],
5081    b: usize,
5082    rows: usize,
5083    cols: usize,
5084    out: &mut [f32],
5085    pool: Option<&Pool>,
5086) {
5087    debug_assert_eq!(out.len(), b * rows);
5088    let gpr = cols / GROUP_SIZE;
5089    let out_addr = SendMut(out.as_mut_ptr());
5090    if a8w8_enabled() {
5091        let acts: Vec<(SplitAct, Vec<i32>)> = (0..b)
5092            .map(|bi| {
5093                let act = split_act(&xs_all[bi * cols..(bi + 1) * cols]);
5094                let gsum = q1_group_sums(&act.xq, gpr);
5095                (act, gsum)
5096            })
5097            .collect();
5098        let acts = &acts;
5099        #[cfg(target_arch = "x86_64")]
5100        let blocked_ok = avx2_enabled()
5101            && std::env::var("CMF_X86_BLOCKED")
5102                .map(|v| v != "0")
5103                .unwrap_or(true);
5104        #[cfg(target_arch = "aarch64")]
5105        let blocked_ok = sdot_enabled()
5106            && std::env::var("CMF_X86_BLOCKED")
5107                .map(|v| v != "0")
5108                .unwrap_or(true);
5109        let run = move |start: usize, end: usize| {
5110            for r in start..end {
5111                let mut bi = 0usize;
5112                // Blocked 1×4: the unpacked bit mask serves four
5113                // activation streams per group.
5114                #[cfg(target_arch = "aarch64")]
5115                if blocked_ok {
5116                    while bi + 4 <= acts.len() {
5117                        let xs = [
5118                            acts[bi].0.xq.as_slice(),
5119                            acts[bi + 1].0.xq.as_slice(),
5120                            acts[bi + 2].0.xq.as_slice(),
5121                            acts[bi + 3].0.xq.as_slice(),
5122                        ];
5123                        let gs = [
5124                            acts[bi].1.as_slice(),
5125                            acts[bi + 1].1.as_slice(),
5126                            acts[bi + 2].1.as_slice(),
5127                            acts[bi + 3].1.as_slice(),
5128                        ];
5129                        let d = unsafe { dot_q1_row_1x4_sdot(bytes, r, gpr, xs, gs) };
5130                        for k in 0..4 {
5131                            let (act, _) = &acts[bi + k];
5132                            let mut acc = d[k] * act.sx;
5133                            for &(j, xv) in &act.outliers {
5134                                let (w, sc) = q1_outlier(bytes, r, gpr, j);
5135                                acc += w * sc * xv;
5136                            }
5137                            // SAFETY: disjoint (bi, r) cells per worker.
5138                            unsafe { *out_addr.at((bi + k) * rows + r) = acc };
5139                        }
5140                        bi += 4;
5141                    }
5142                }
5143                #[cfg(target_arch = "x86_64")]
5144                if blocked_ok {
5145                    while bi + 4 <= acts.len() {
5146                        let xs = [
5147                            acts[bi].0.xq.as_slice(),
5148                            acts[bi + 1].0.xq.as_slice(),
5149                            acts[bi + 2].0.xq.as_slice(),
5150                            acts[bi + 3].0.xq.as_slice(),
5151                        ];
5152                        let gs = [
5153                            acts[bi].1.as_slice(),
5154                            acts[bi + 1].1.as_slice(),
5155                            acts[bi + 2].1.as_slice(),
5156                            acts[bi + 3].1.as_slice(),
5157                        ];
5158                        let d = unsafe {
5159                            if vnni_tiles_enabled() {
5160                                dot_q1_row_1x4_vnni(bytes, r, gpr, xs, gs)
5161                            } else {
5162                                dot_q1_row_1x4_avx2(bytes, r, gpr, xs, gs)
5163                            }
5164                        };
5165                        for k in 0..4 {
5166                            let (act, _) = &acts[bi + k];
5167                            let mut acc = d[k] * act.sx;
5168                            for &(j, xv) in &act.outliers {
5169                                let (w, sc) = q1_outlier(bytes, r, gpr, j);
5170                                acc += w * sc * xv;
5171                            }
5172                            // SAFETY: disjoint (bi, r) cells per worker.
5173                            unsafe { *out_addr.at((bi + k) * rows + r) = acc };
5174                        }
5175                        bi += 4;
5176                    }
5177                }
5178                while bi < acts.len() {
5179                    let (act, gsum) = &acts[bi];
5180                    let mut acc = dot_q1_row_i8(bytes, r, gpr, &act.xq, gsum) * act.sx;
5181                    for &(j, xv) in &act.outliers {
5182                        let (w, s) = q1_outlier(bytes, r, gpr, j);
5183                        acc += w * s * xv;
5184                    }
5185                    // SAFETY: disjoint (bi, r) cells per worker range.
5186                    unsafe { *out_addr.at(bi * rows + r) = acc };
5187                    bi += 1;
5188                }
5189            }
5190        };
5191        dispatch_rows(pool, rows, &run);
5192        return;
5193    }
5194    let run = move |start: usize, end: usize| {
5195        for r in start..end {
5196            for bi in 0..b {
5197                let x = &xs_all[bi * cols..(bi + 1) * cols];
5198                // SAFETY: disjoint (bi, r) cells per worker range.
5199                unsafe { *out_addr.at(bi * rows + r) = q1_row_exact(bytes, r, gpr, x) };
5200            }
5201        }
5202    };
5203    dispatch_rows(pool, rows, &run);
5204}
5205
5206/// Fused q4_block matvec straight from the mapped bytes. SDOT path when
5207/// dotprod is available (port of vmfcore `dot_q4_block_sdot`, measured
5208/// +23% on q4 decode): nibbles → centered i8, int8×int8 `sdot` per
5209/// 32-group, exact outlier correction — the same A8W8 contract as q8.
5210/// `CMF_SDOT=0` keeps the exact scalar path.
5211fn q4matvec(
5212    bytes: &[u8],
5213    x: &[f32],
5214    rows: usize,
5215    cols: usize,
5216    out: &mut [f32],
5217    pool: Option<&Pool>,
5218) {
5219    debug_assert_eq!(out.len(), rows);
5220    let (packed, scales) = q4_split(bytes, rows, cols);
5221    let gpr = cols / GROUP_SIZE;
5222    let out_addr = SendMut(out.as_mut_ptr());
5223
5224    if a8w8_enabled() {
5225        let act = split_act(x);
5226        let run = move |start: usize, end: usize| {
5227            q4_range_a8w8(packed, scales, gpr, cols, &act, out_addr, start, end)
5228        };
5229        dispatch_rows(pool, rows, &run);
5230        return;
5231    }
5232
5233    let run =
5234        move |start: usize, end: usize| q4_range_f32(packed, scales, gpr, x, out_addr, start, end);
5235    dispatch_rows(pool, rows, &run);
5236}
5237
5238/// One q4 row via the A8W8 int8 path — SDOT on ARM, AVX2 maddubs on
5239/// x86 (scalar fallback is unreachable: callers gate on a8w8_enabled).
5240#[inline]
5241#[allow(unreachable_code)]
5242/// One UNPACKED q4 row (centered i8 in `buf`) against four activation
5243/// streams: the 32-byte weight chunk and its abs() load once per group,
5244/// the per-group f16 scale decodes once — four maddubs+reduce chains
5245/// instead of four full (load, abs, dot) rounds.
5246#[cfg(target_arch = "x86_64")]
5247#[target_feature(enable = "avx2")]
5248unsafe fn dot_q4b_row_1x4_avx2(
5249    buf: &[u8],
5250    scales: &[u8],
5251    g0: usize,
5252    gpr: usize,
5253    xs: [&[i8]; 4],
5254) -> [f32; 4] {
5255    // SAFETY: callers uphold buffer contracts (buf.len() == gpr·32).
5256    unsafe {
5257        use core::arch::x86_64::*;
5258        let ones = _mm256_set1_epi16(1);
5259        let mut acc = [0f32; 4];
5260        for gi in 0..gpr {
5261            let s = f16_to_f32(u16::from_le_bytes([
5262                scales[(g0 + gi) * 2],
5263                scales[(g0 + gi) * 2 + 1],
5264            ]));
5265            let w = _mm256_loadu_si256(buf.as_ptr().add(gi * GROUP_SIZE) as *const __m256i);
5266            let aw = _mm256_abs_epi8(w);
5267            for (k, xq) in xs.iter().enumerate() {
5268                let x = _mm256_loadu_si256(xq.as_ptr().add(gi * GROUP_SIZE) as *const __m256i);
5269                let p16 = _mm256_maddubs_epi16(aw, _mm256_sign_epi8(x, w));
5270                let d = _mm256_madd_epi16(p16, ones);
5271                let hi128 = _mm256_extracti128_si256::<1>(d);
5272                let s128 = _mm_add_epi32(_mm256_castsi256_si128(d), hi128);
5273                let s64 = _mm_add_epi32(s128, _mm_srli_si128::<8>(s128));
5274                let s32 = _mm_add_epi32(s64, _mm_srli_si128::<4>(s64));
5275                acc[k] += _mm_cvtsi128_si32(s32) as f32 * s;
5276            }
5277        }
5278        acc
5279    }
5280}
5281
5282/// VNNI twin of `dot_q4b_row_1x4_avx2` (see `dpbusd_hsum`).
5283#[cfg(target_arch = "x86_64")]
5284#[target_feature(enable = "avx2,avx512f,avx512bw,avx512vl,avx512vnni")]
5285unsafe fn dot_q4b_row_1x4_vnni(
5286    buf: &[u8],
5287    scales: &[u8],
5288    g0: usize,
5289    gpr: usize,
5290    xs: [&[i8]; 4],
5291) -> [f32; 4] {
5292    // SAFETY: callers uphold buffer contracts (buf.len() == gpr·32).
5293    unsafe {
5294        use core::arch::x86_64::*;
5295        let mut acc = [0f32; 4];
5296        for gi in 0..gpr {
5297            let s = f16_to_f32(u16::from_le_bytes([
5298                scales[(g0 + gi) * 2],
5299                scales[(g0 + gi) * 2 + 1],
5300            ]));
5301            let w = _mm256_loadu_si256(buf.as_ptr().add(gi * GROUP_SIZE) as *const __m256i);
5302            let aw = _mm256_abs_epi8(w);
5303            for (k, xq) in xs.iter().enumerate() {
5304                let x = _mm256_loadu_si256(xq.as_ptr().add(gi * GROUP_SIZE) as *const __m256i);
5305                let d = dpbusd_hsum(aw, _mm256_sign_epi8(x, w));
5306                acc[k] += d as f32 * s;
5307            }
5308        }
5309        acc
5310    }
5311}
5312
5313/// The vbit flavor of the blocked 1×4: the per-activation A8W8 scale
5314/// folds in PER GROUP as `(d·sx)·s` — bit-matching the single-matvec
5315/// accumulation order (the q4_block flavor applies sx once at the end,
5316/// matching ITS single path; the two conventions are historical and
5317/// each blocked leg must mirror its own).
5318#[cfg(target_arch = "x86_64")]
5319#[target_feature(enable = "avx2")]
5320unsafe fn dot_q4b_row_1x4_sx_avx2(
5321    buf: &[u8],
5322    scales: &[u8],
5323    g0: usize,
5324    gpr: usize,
5325    xs: [&[i8]; 4],
5326    sxs: [f32; 4],
5327) -> [f32; 4] {
5328    // SAFETY: callers uphold buffer contracts (buf.len() == gpr·32).
5329    unsafe {
5330        use core::arch::x86_64::*;
5331        let ones = _mm256_set1_epi16(1);
5332        let mut acc = [0f32; 4];
5333        for gi in 0..gpr {
5334            let s = f16_to_f32(u16::from_le_bytes([
5335                scales[(g0 + gi) * 2],
5336                scales[(g0 + gi) * 2 + 1],
5337            ]));
5338            let w = _mm256_loadu_si256(buf.as_ptr().add(gi * GROUP_SIZE) as *const __m256i);
5339            let aw = _mm256_abs_epi8(w);
5340            for (k, xq) in xs.iter().enumerate() {
5341                let x = _mm256_loadu_si256(xq.as_ptr().add(gi * GROUP_SIZE) as *const __m256i);
5342                let p16 = _mm256_maddubs_epi16(aw, _mm256_sign_epi8(x, w));
5343                let d = _mm256_madd_epi16(p16, ones);
5344                let hi128 = _mm256_extracti128_si256::<1>(d);
5345                let s128 = _mm_add_epi32(_mm256_castsi256_si128(d), hi128);
5346                let s64 = _mm_add_epi32(s128, _mm_srli_si128::<8>(s128));
5347                let s32 = _mm_add_epi32(s64, _mm_srli_si128::<4>(s64));
5348                acc[k] += (_mm_cvtsi128_si32(s32) as f32 * sxs[k]) * s;
5349            }
5350        }
5351        acc
5352    }
5353}
5354
5355/// VNNI twin of `dot_q4b_row_1x4_sx_avx2` (see `dpbusd_hsum`; the
5356/// per-group `(d·sx)·s` fold mirrors the vbit single path).
5357#[cfg(target_arch = "x86_64")]
5358#[target_feature(enable = "avx2,avx512f,avx512bw,avx512vl,avx512vnni")]
5359unsafe fn dot_q4b_row_1x4_sx_vnni(
5360    buf: &[u8],
5361    scales: &[u8],
5362    g0: usize,
5363    gpr: usize,
5364    xs: [&[i8]; 4],
5365    sxs: [f32; 4],
5366) -> [f32; 4] {
5367    // SAFETY: callers uphold buffer contracts (buf.len() == gpr·32).
5368    unsafe {
5369        use core::arch::x86_64::*;
5370        let mut acc = [0f32; 4];
5371        for gi in 0..gpr {
5372            let s = f16_to_f32(u16::from_le_bytes([
5373                scales[(g0 + gi) * 2],
5374                scales[(g0 + gi) * 2 + 1],
5375            ]));
5376            let w = _mm256_loadu_si256(buf.as_ptr().add(gi * GROUP_SIZE) as *const __m256i);
5377            let aw = _mm256_abs_epi8(w);
5378            for (k, xq) in xs.iter().enumerate() {
5379                let x = _mm256_loadu_si256(xq.as_ptr().add(gi * GROUP_SIZE) as *const __m256i);
5380                let d = dpbusd_hsum(aw, _mm256_sign_epi8(x, w));
5381                acc[k] += (d as f32 * sxs[k]) * s;
5382            }
5383        }
5384        acc
5385    }
5386}
5387
5388#[allow(unreachable_code)]
5389fn dot_q4_row_i8(packed: &[u8], scales: &[u8], g0: usize, gpr: usize, xq: &[i8]) -> f32 {
5390    #[cfg(target_arch = "aarch64")]
5391    unsafe {
5392        return dot_q4_row_sdot(packed, scales, g0, gpr, xq);
5393    }
5394    #[cfg(target_arch = "x86_64")]
5395    unsafe {
5396        return dot_q4_row_avx2(packed, scales, g0, gpr, xq);
5397    }
5398    let mut acc = 0f32;
5399    for gi in 0..gpr {
5400        let g = g0 + gi;
5401        let s = f16_to_f32(u16::from_le_bytes([scales[g * 2], scales[g * 2 + 1]]));
5402        let mut d = 0i32;
5403        for (k, &b) in packed[g * 16..(g + 1) * 16].iter().enumerate() {
5404            d += ((b & 0x0F) as i32 - 8) * xq[gi * GROUP_SIZE + k * 2] as i32
5405                + (((b >> 4) & 0x0F) as i32 - 8) * xq[gi * GROUP_SIZE + k * 2 + 1] as i32;
5406        }
5407        acc += d as f32 * s;
5408    }
5409    acc
5410}
5411
5412/// Two-activation q4 row via the A8W8 int8 path (see `dot_q4_row_i8`).
5413#[inline]
5414#[allow(unreachable_code)]
5415fn dot_q4_row_i8_2(
5416    packed: &[u8],
5417    scales: &[u8],
5418    g0: usize,
5419    gpr: usize,
5420    xq1: &[i8],
5421    xq2: &[i8],
5422) -> (f32, f32) {
5423    #[cfg(target_arch = "aarch64")]
5424    unsafe {
5425        return dot_q4_row_sdot2(packed, scales, g0, gpr, xq1, xq2);
5426    }
5427    #[cfg(target_arch = "x86_64")]
5428    unsafe {
5429        return dot_q4_row_avx2_2(packed, scales, g0, gpr, xq1, xq2);
5430    }
5431    (
5432        dot_q4_row_i8(packed, scales, g0, gpr, xq1),
5433        dot_q4_row_i8(packed, scales, g0, gpr, xq2),
5434    )
5435}
5436
5437/// One q4 row range via SDOT (kernel body of `q4matvec`, extracted so
5438/// multi-matrix jobs can drive it for several tensors in one dispatch).
5439#[allow(clippy::too_many_arguments)]
5440fn q4_range_a8w8(
5441    packed: &[u8],
5442    scales: &[u8],
5443    gpr: usize,
5444    cols: usize,
5445    act: &SplitAct,
5446    out: SendMut,
5447    start: usize,
5448    end: usize,
5449) {
5450    for r in start..end {
5451        let mut acc = dot_q4_row_i8(packed, scales, r * gpr, gpr, &act.xq) * act.sx;
5452        // xq is zeroed at outlier slots — add the exact terms.
5453        for &(j, xv) in &act.outliers {
5454            let flat = r * cols + j;
5455            let byte = packed[flat / 2];
5456            let nib = if flat & 1 == 0 {
5457                byte & 0x0F
5458            } else {
5459                byte >> 4
5460            };
5461            let s = f16_to_f32(u16::from_le_bytes([
5462                scales[(flat / GROUP_SIZE) * 2],
5463                scales[(flat / GROUP_SIZE) * 2 + 1],
5464            ]));
5465            acc += ((nib as i32 - 8) as f32) * s * xv;
5466        }
5467        // SAFETY: disjoint row ranges per worker.
5468        unsafe { *out.at(r) = acc };
5469    }
5470}
5471
5472/// Two-input q4 row range via the A8W8 int8 path — kernel body of
5473/// `q4matvec2`, extracted for pair multi-matrix jobs.
5474#[allow(clippy::too_many_arguments)]
5475fn q4_range2_a8w8(
5476    packed: &[u8],
5477    scales: &[u8],
5478    gpr: usize,
5479    cols: usize,
5480    a1: &SplitAct,
5481    a2: &SplitAct,
5482    p1: SendMut,
5483    p2: SendMut,
5484    start: usize,
5485    end: usize,
5486) {
5487    for r in start..end {
5488        let (s1, s2) = dot_q4_row_i8_2(packed, scales, r * gpr, gpr, &a1.xq, &a2.xq);
5489        let mut acc1 = s1 * a1.sx;
5490        let mut acc2 = s2 * a2.sx;
5491        // xq is zeroed at outlier slots — add the exact terms.
5492        let fix = |outliers: &[(usize, f32)], acc: &mut f32| {
5493            for &(j, xv) in outliers {
5494                let flat = r * cols + j;
5495                let byte = packed[flat / 2];
5496                let nib = if flat & 1 == 0 {
5497                    byte & 0x0F
5498                } else {
5499                    byte >> 4
5500                };
5501                let s = f16_to_f32(u16::from_le_bytes([
5502                    scales[(flat / GROUP_SIZE) * 2],
5503                    scales[(flat / GROUP_SIZE) * 2 + 1],
5504                ]));
5505                *acc += ((nib as i32 - 8) as f32) * s * xv;
5506            }
5507        };
5508        fix(&a1.outliers, &mut acc1);
5509        fix(&a2.outliers, &mut acc2);
5510        // SAFETY: disjoint row ranges per worker.
5511        unsafe {
5512            *p1.at(r) = acc1;
5513            *p2.at(r) = acc2;
5514        }
5515    }
5516}
5517
5518/// Exact scalar q4 row range (same extraction, non-SDOT path).
5519fn q4_range_f32(
5520    packed: &[u8],
5521    scales: &[u8],
5522    gpr: usize,
5523    x: &[f32],
5524    out: SendMut,
5525    start: usize,
5526    end: usize,
5527) {
5528    for r in start..end {
5529        let mut acc = 0f32;
5530        for gi in 0..gpr {
5531            let g = r * gpr + gi;
5532            let s = f16_to_f32(u16::from_le_bytes([scales[g * 2], scales[g * 2 + 1]]));
5533            let pk = &packed[g * 16..(g + 1) * 16];
5534            let xg = &x[gi * GROUP_SIZE..(gi + 1) * GROUP_SIZE];
5535            let mut ga = 0f32;
5536            for (k, &b) in pk.iter().enumerate() {
5537                ga += ((b & 0x0F) as f32 - 8.0) * xg[k * 2]
5538                    + (((b >> 4) & 0x0F) as f32 - 8.0) * xg[k * 2 + 1];
5539            }
5540            acc += ga * s;
5541        }
5542        // SAFETY: disjoint row ranges per worker.
5543        unsafe { *out.at(r) = acc };
5544    }
5545}
5546
5547/// Fused two-input q4 matvec: nibbles are unpacked ONCE per group and
5548/// dotted against both activations (was: two full matvecs — double
5549/// weight traffic). Per-lane math matches `q4matvec` exactly.
5550#[allow(clippy::too_many_arguments)]
5551fn q4matvec2(
5552    bytes: &[u8],
5553    x1: &[f32],
5554    x2: &[f32],
5555    rows: usize,
5556    cols: usize,
5557    o1: &mut [f32],
5558    o2: &mut [f32],
5559    pool: Option<&Pool>,
5560) {
5561    debug_assert_eq!(o1.len(), rows);
5562    debug_assert_eq!(o2.len(), rows);
5563    let (packed, scales) = q4_split(bytes, rows, cols);
5564    let gpr = cols / GROUP_SIZE;
5565
5566    if a8w8_enabled() {
5567        let a1 = split_act(x1);
5568        let a2 = split_act(x2);
5569        let p1 = SendMut(o1.as_mut_ptr());
5570        let p2 = SendMut(o2.as_mut_ptr());
5571        let run = move |start: usize, end: usize| {
5572            q4_range2_a8w8(packed, scales, gpr, cols, &a1, &a2, p1, p2, start, end)
5573        };
5574        dispatch_rows(pool, rows, &run);
5575        return;
5576    }
5577
5578    let p1 = SendMut(o1.as_mut_ptr());
5579    let p2 = SendMut(o2.as_mut_ptr());
5580    let run = move |start: usize, end: usize| {
5581        q4_range2_f32(packed, scales, gpr, x1, x2, p1, p2, start, end)
5582    };
5583    dispatch_rows(pool, rows, &run);
5584}
5585
5586/// Two-input exact scalar q4 row range (same extraction).
5587#[allow(clippy::too_many_arguments)]
5588fn q4_range2_f32(
5589    packed: &[u8],
5590    scales: &[u8],
5591    gpr: usize,
5592    x1: &[f32],
5593    x2: &[f32],
5594    p1: SendMut,
5595    p2: SendMut,
5596    start: usize,
5597    end: usize,
5598) {
5599    for r in start..end {
5600        let (mut acc1, mut acc2) = (0f32, 0f32);
5601        for gi in 0..gpr {
5602            let g = r * gpr + gi;
5603            let s = f16_to_f32(u16::from_le_bytes([scales[g * 2], scales[g * 2 + 1]]));
5604            let pk = &packed[g * 16..(g + 1) * 16];
5605            let x1g = &x1[gi * GROUP_SIZE..(gi + 1) * GROUP_SIZE];
5606            let x2g = &x2[gi * GROUP_SIZE..(gi + 1) * GROUP_SIZE];
5607            let (mut g1, mut g2) = (0f32, 0f32);
5608            for (k, &b) in pk.iter().enumerate() {
5609                let wl = (b & 0x0F) as f32 - 8.0;
5610                let wh = ((b >> 4) & 0x0F) as f32 - 8.0;
5611                g1 += wl * x1g[k * 2] + wh * x1g[k * 2 + 1];
5612                g2 += wl * x2g[k * 2] + wh * x2g[k * 2 + 1];
5613            }
5614            acc1 += g1 * s;
5615            acc2 += g2 * s;
5616        }
5617        // SAFETY: disjoint row ranges per worker.
5618        unsafe {
5619            *p1.at(r) = acc1;
5620            *p2.at(r) = acc2;
5621        }
5622    }
5623}
5624
5625thread_local! {
5626    /// Per-worker decoded-row scratch for the batched q4/vbit kernels
5627    /// (centered i8 for SDOT, f32 for the exact/scalar paths).
5628    static ROW_I8: std::cell::RefCell<Vec<u8>> = const { std::cell::RefCell::new(Vec::new()) };
5629    static ROW_F32: std::cell::RefCell<Vec<f32>> = const { std::cell::RefCell::new(Vec::new()) };
5630}
5631
5632/// Batched q4 matmat: each weight row is unpacked from the mmap ONCE
5633/// and dotted against ALL b activations (prefill used to fall back to b
5634/// full matvecs — b× weight traffic and b× nibble decode). Per-position
5635/// math matches `q4matvec` exactly: same group order, same accumulation.
5636/// `out` is row-major [b, rows] like `qmatmat`.
5637#[allow(clippy::too_many_arguments)]
5638fn q4matmat(
5639    bytes: &[u8],
5640    xs_all: &[f32],
5641    b: usize,
5642    rows: usize,
5643    cols: usize,
5644    out: &mut [f32],
5645    pool: Option<&Pool>,
5646) {
5647    debug_assert_eq!(xs_all.len(), b * cols);
5648    debug_assert_eq!(out.len(), b * rows);
5649    let (packed, scales) = q4_split(bytes, rows, cols);
5650    let gpr = cols / GROUP_SIZE;
5651    let gscale = |g: usize| f16_to_f32(u16::from_le_bytes([scales[g * 2], scales[g * 2 + 1]]));
5652
5653    if a8w8_enabled() {
5654        let acts: Vec<SplitAct> = (0..b)
5655            .map(|bi| split_act(&xs_all[bi * cols..(bi + 1) * cols]))
5656            .collect();
5657        let acts = &acts;
5658        let out_addr = SendMut(out.as_mut_ptr());
5659        let run = move |start: usize, end: usize| {
5660            ROW_I8.with(|rb| {
5661                let mut buf = rb.borrow_mut();
5662                buf.resize(cols, 0);
5663                for r in start..end {
5664                    // Unpack the row's nibbles to centered i8 once
5665                    // (element 2k = low nibble, 2k+1 = high — flat order,
5666                    // same as dot_q4_row_sdot's zip).
5667                    for gi in 0..gpr {
5668                        let g = r * gpr + gi;
5669                        for (k, &bt) in packed[g * 16..(g + 1) * 16].iter().enumerate() {
5670                            buf[gi * GROUP_SIZE + k * 2] = ((bt & 0x0F) as i32 - 8) as i8 as u8;
5671                            buf[gi * GROUP_SIZE + k * 2 + 1] =
5672                                (((bt >> 4) & 0x0F) as i32 - 8) as i8 as u8;
5673                        }
5674                    }
5675                    let mut bi = 0usize;
5676                    #[cfg(target_arch = "x86_64")]
5677                    if avx2_enabled()
5678                        && std::env::var("CMF_X86_BLOCKED")
5679                            .map(|v| v != "0")
5680                            .unwrap_or(true)
5681                    {
5682                        while bi + 4 <= acts.len() {
5683                            let xs = [
5684                                acts[bi].xq.as_slice(),
5685                                acts[bi + 1].xq.as_slice(),
5686                                acts[bi + 2].xq.as_slice(),
5687                                acts[bi + 3].xq.as_slice(),
5688                            ];
5689                            let d = unsafe {
5690                                if vnni_tiles_enabled() {
5691                                    dot_q4b_row_1x4_vnni(&buf, scales, r * gpr, gpr, xs)
5692                                } else {
5693                                    dot_q4b_row_1x4_avx2(&buf, scales, r * gpr, gpr, xs)
5694                                }
5695                            };
5696                            for k in 0..4 {
5697                                let act = &acts[bi + k];
5698                                let mut acc = d[k] * act.sx;
5699                                for &(j, xv) in &act.outliers {
5700                                    acc += (buf[j] as i8) as f32
5701                                        * gscale((r * cols + j) / GROUP_SIZE)
5702                                        * xv;
5703                                }
5704                                // SAFETY: disjoint (bi, r) cells per worker.
5705                                unsafe { *out_addr.at((bi + k) * rows + r) = acc };
5706                            }
5707                            bi += 4;
5708                        }
5709                    }
5710                    while bi < acts.len() {
5711                        let act = &acts[bi];
5712                        let mut acc = 0f32;
5713                        for gi in 0..gpr {
5714                            let d = dot_i8_i8(
5715                                &buf[gi * GROUP_SIZE..(gi + 1) * GROUP_SIZE],
5716                                &act.xq[gi * GROUP_SIZE..(gi + 1) * GROUP_SIZE],
5717                            );
5718                            acc += d as f32 * gscale(r * gpr + gi);
5719                        }
5720                        acc *= act.sx;
5721                        // xq is zeroed at outlier slots — exact terms.
5722                        for &(j, xv) in &act.outliers {
5723                            acc += (buf[j] as i8) as f32 * gscale((r * cols + j) / GROUP_SIZE) * xv;
5724                        }
5725                        // SAFETY: disjoint (bi, r) cells per worker row range.
5726                        unsafe { *out_addr.at(bi * rows + r) = acc };
5727                        bi += 1;
5728                    }
5729                }
5730            })
5731        };
5732        dispatch_rows(pool, rows, &run);
5733        return;
5734    }
5735
5736    let out_addr = SendMut(out.as_mut_ptr());
5737    let run = move |start: usize, end: usize| {
5738        ROW_F32.with(|rb| {
5739            let mut buf = rb.borrow_mut();
5740            buf.resize(cols, 0.0);
5741            for r in start..end {
5742                // Decode raw (nib − 8) values once; scales stay per-group
5743                // so the accumulation order matches q4matvec bit-for-bit.
5744                for gi in 0..gpr {
5745                    let g = r * gpr + gi;
5746                    for (k, &bt) in packed[g * 16..(g + 1) * 16].iter().enumerate() {
5747                        buf[gi * GROUP_SIZE + k * 2] = (bt & 0x0F) as f32 - 8.0;
5748                        buf[gi * GROUP_SIZE + k * 2 + 1] = ((bt >> 4) & 0x0F) as f32 - 8.0;
5749                    }
5750                }
5751                for bi in 0..b {
5752                    let x = &xs_all[bi * cols..(bi + 1) * cols];
5753                    let mut acc = 0f32;
5754                    for gi in 0..gpr {
5755                        let mut ga = 0f32;
5756                        // Pairwise (lo + hi) addition, matching
5757                        // q4matvec's `ga += lo·x + hi·x` shape exactly —
5758                        // a flat one-per-element loop rounds differently
5759                        // and broke bit-parity on the scalar (x86) path.
5760                        for k in 0..GROUP_SIZE / 2 {
5761                            let e = gi * GROUP_SIZE + k * 2;
5762                            ga += buf[e] * x[e] + buf[e + 1] * x[e + 1];
5763                        }
5764                        acc += ga * gscale(r * gpr + gi);
5765                    }
5766                    // SAFETY: disjoint (bi, r) cells per worker row range.
5767                    unsafe { *out_addr.at(bi * rows + r) = acc };
5768                }
5769            }
5770        })
5771    };
5772    dispatch_rows(pool, rows, &run);
5773}
5774
5775/// Batched vbit matmat: each variable-bit row is decoded from the mmap
5776/// ONCE for the whole microbatch. Same per-position math as
5777/// `vbitmatvec` (SDOT A8W8 with exact outliers / exact f32 for b=8 rows
5778/// and the scalar path).
5779#[allow(clippy::too_many_arguments)]
5780fn vbitmatmat(
5781    bytes: &[u8],
5782    offsets: &[usize],
5783    xs_all: &[f32],
5784    b: usize,
5785    rows: usize,
5786    cols: usize,
5787    out: &mut [f32],
5788    pool: Option<&Pool>,
5789) {
5790    debug_assert_eq!(xs_all.len(), b * cols);
5791    debug_assert_eq!(out.len(), b * rows);
5792    debug_assert_eq!(offsets.len(), rows + 1);
5793    let ng = cols / GROUP_SIZE;
5794    let bits = &bytes[..rows];
5795    let sc_off = rows;
5796    let gscale = |r: usize, g: usize| {
5797        let so = (r * ng + g) * 2;
5798        f16_to_f32(u16::from_le_bytes([
5799            bytes[sc_off + so],
5800            bytes[sc_off + so + 1],
5801        ]))
5802    };
5803
5804    // Decode row r's raw (u − L) values into `dst` (f32, unscaled).
5805    let decode_f32 = |r: usize, dst: &mut [f32]| {
5806        let bw = bits[r] as usize;
5807        let l = ((1i32 << (bw - 1)) - 1) as f32;
5808        let data = &bytes[offsets[r]..offsets[r + 1]];
5809        let (mut acc, mut nbits, mut idx) = (0u64, 0usize, 0usize);
5810        for d in dst.iter_mut() {
5811            while nbits < bw {
5812                acc = (acc << 8) | data[idx] as u64;
5813                idx += 1;
5814                nbits += 8;
5815            }
5816            let u = ((acc >> (nbits - bw)) & ((1u64 << bw) - 1)) as f32;
5817            nbits -= bw;
5818            *d = u - l;
5819        }
5820    };
5821
5822    if a8w8_enabled() {
5823        let acts: Vec<SplitAct> = (0..b)
5824            .map(|bi| split_act(&xs_all[bi * cols..(bi + 1) * cols]))
5825            .collect();
5826        let acts = &acts;
5827        let out_addr = SendMut(out.as_mut_ptr());
5828        let run = move |start: usize, end: usize| {
5829            for r in start..end {
5830                let bw = bits[r] as usize;
5831                if bw == 8 {
5832                    // u−L reaches 128 → no i8 path; decode once, exact
5833                    // f32 dots for every position (same as vbitmatvec).
5834                    ROW_F32.with(|rb| {
5835                        let mut buf = rb.borrow_mut();
5836                        buf.resize(cols, 0.0);
5837                        decode_f32(r, &mut buf);
5838                        for bi in 0..b {
5839                            let x = &xs_all[bi * cols..(bi + 1) * cols];
5840                            let mut dot = 0f32;
5841                            for g in 0..ng {
5842                                let mut gd = 0f32;
5843                                for k in 0..GROUP_SIZE {
5844                                    gd += buf[g * GROUP_SIZE + k] * x[g * GROUP_SIZE + k];
5845                                }
5846                                dot += gd * gscale(r, g);
5847                            }
5848                            // SAFETY: disjoint (bi, r) cells per worker range.
5849                            unsafe { *out_addr.at(bi * rows + r) = dot };
5850                        }
5851                    });
5852                    continue;
5853                }
5854                let l = (1i32 << (bw - 1)) - 1;
5855                let data = &bytes[offsets[r]..offsets[r + 1]];
5856                ROW_I8.with(|rb| {
5857                    let mut buf = rb.borrow_mut();
5858                    buf.resize(cols, 0);
5859                    #[inline(always)]
5860                    fn fill<const B: usize>(data: &[u8], l: i32, buf: &mut [u8]) {
5861                        for (blk, chunk) in buf.chunks_exact_mut(8).enumerate() {
5862                            let u = unpack8::<B>(&data[blk * B..]);
5863                            for k in 0..8 {
5864                                chunk[k] = (u[k] - l) as i8 as u8;
5865                            }
5866                        }
5867                    }
5868                    match bw {
5869                        3 => fill::<3>(data, l, &mut buf),
5870                        4 => vbit_fill4(data, &mut buf),
5871                        5 => fill::<5>(data, l, &mut buf),
5872                        6 => fill::<6>(data, l, &mut buf),
5873                        _ => unreachable!("vbit bit-width {bw} (validated at load)"),
5874                    }
5875                    let mut bi = 0usize;
5876                    // The vbit scale table shares q4_block's layout
5877                    // (contiguous f16 per (row·ng + g)), so the same
5878                    // blocked 1×4 kernel serves the decoded row.
5879                    #[cfg(target_arch = "x86_64")]
5880                    if avx2_enabled()
5881                        && std::env::var("CMF_X86_BLOCKED")
5882                            .map(|v| v != "0")
5883                            .unwrap_or(true)
5884                    {
5885                        while bi + 4 <= acts.len() {
5886                            let xs = [
5887                                acts[bi].xq.as_slice(),
5888                                acts[bi + 1].xq.as_slice(),
5889                                acts[bi + 2].xq.as_slice(),
5890                                acts[bi + 3].xq.as_slice(),
5891                            ];
5892                            let sxs = [
5893                                acts[bi].sx,
5894                                acts[bi + 1].sx,
5895                                acts[bi + 2].sx,
5896                                acts[bi + 3].sx,
5897                            ];
5898                            let d = unsafe {
5899                                if vnni_tiles_enabled() {
5900                                    dot_q4b_row_1x4_sx_vnni(
5901                                        &buf,
5902                                        &bytes[sc_off..],
5903                                        r * ng,
5904                                        ng,
5905                                        xs,
5906                                        sxs,
5907                                    )
5908                                } else {
5909                                    dot_q4b_row_1x4_sx_avx2(
5910                                        &buf,
5911                                        &bytes[sc_off..],
5912                                        r * ng,
5913                                        ng,
5914                                        xs,
5915                                        sxs,
5916                                    )
5917                                }
5918                            };
5919                            for k in 0..4 {
5920                                let act = &acts[bi + k];
5921                                let mut dot = d[k];
5922                                for &(j, xv) in &act.outliers {
5923                                    dot += (buf[j] as i8) as f32 * gscale(r, j / GROUP_SIZE) * xv;
5924                                }
5925                                // SAFETY: disjoint (bi, r) cells per worker.
5926                                unsafe { *out_addr.at((bi + k) * rows + r) = dot };
5927                            }
5928                            bi += 4;
5929                        }
5930                    }
5931                    while bi < acts.len() {
5932                        let act = &acts[bi];
5933                        let mut dot = 0f32;
5934                        for g in 0..ng {
5935                            let d = dot_i8_i8(
5936                                &buf[g * GROUP_SIZE..(g + 1) * GROUP_SIZE],
5937                                &act.xq[g * GROUP_SIZE..(g + 1) * GROUP_SIZE],
5938                            ) as f32
5939                                * act.sx;
5940                            dot += d * gscale(r, g);
5941                        }
5942                        for &(j, xv) in &act.outliers {
5943                            dot += (buf[j] as i8) as f32 * gscale(r, j / GROUP_SIZE) * xv;
5944                        }
5945                        // SAFETY: disjoint (bi, r) cells per worker range.
5946                        unsafe { *out_addr.at(bi * rows + r) = dot };
5947                        bi += 1;
5948                    }
5949                });
5950            }
5951        };
5952        dispatch_rows(pool, rows, &run);
5953        return;
5954    }
5955
5956    let out_addr = SendMut(out.as_mut_ptr());
5957    let run = move |start: usize, end: usize| {
5958        ROW_F32.with(|rb| {
5959            let mut buf = rb.borrow_mut();
5960            buf.resize(cols, 0.0);
5961            for r in start..end {
5962                decode_f32(r, &mut buf);
5963                for bi in 0..b {
5964                    let x = &xs_all[bi * cols..(bi + 1) * cols];
5965                    let mut dot = 0f32;
5966                    for g in 0..ng {
5967                        let mut gd = 0f32;
5968                        for k in 0..GROUP_SIZE {
5969                            gd += buf[g * GROUP_SIZE + k] * x[g * GROUP_SIZE + k];
5970                        }
5971                        dot += gd * gscale(r, g);
5972                    }
5973                    // SAFETY: disjoint (bi, r) cells per worker range.
5974                    unsafe { *out_addr.at(bi * rows + r) = dot };
5975                }
5976            }
5977        })
5978    };
5979    dispatch_rows(pool, rows, &run);
5980}
5981
5982/// Build a GPU batch job for a q8-family mapped tensor (primary
5983/// shard): prescaled input + directory coordinates. None → not
5984/// GPU-eligible, caller stays on the CPU.
5985pub(crate) fn gpu_batch_job<'a>(
5986    t: &'a QTensor,
5987    x: &[f32],
5988) -> Option<(std::sync::Arc<CmfModel>, crate::gpu::BatchJob<'a>)> {
5989    match t {
5990        QTensor::Mapped {
5991            model,
5992            idx,
5993            dtype: dt @ (TensorDtype::Q8Row | TensorDtype::Q8_2f),
5994            rows,
5995            cols,
5996            row_scale,
5997            col_field,
5998            ..
5999        } => Some((
6000            model.clone(),
6001            crate::gpu::BatchJob {
6002                idx: *idx,
6003                rows: *rows,
6004                cols: *cols,
6005                row_scale,
6006                xs: prescale(x, col_field, *dt).into_owned(),
6007                q1: false,
6008            },
6009        )),
6010        // q1: raw f32 activations, tile-embedded scales.
6011        QTensor::Mapped {
6012            model,
6013            idx,
6014            dtype: TensorDtype::Q1,
6015            rows,
6016            cols,
6017            ..
6018        } => Some((
6019            model.clone(),
6020            crate::gpu::BatchJob {
6021                idx: *idx,
6022                rows: *rows,
6023                cols: *cols,
6024                row_scale: &[],
6025                xs: x.to_vec(),
6026                q1: true,
6027            },
6028        )),
6029        _ => None,
6030    }
6031}
6032
6033thread_local! {
6034    static PRESCALE_BUF1: std::cell::RefCell<Vec<f32>> = const { std::cell::RefCell::new(Vec::new()) };
6035    static PRESCALE_BUF2: std::cell::RefCell<Vec<f32>> = const { std::cell::RefCell::new(Vec::new()) };
6036}
6037
6038pub(crate) fn prescale<'a>(
6039    x: &'a [f32],
6040    col_field: &[f32],
6041    dtype: TensorDtype,
6042) -> std::borrow::Cow<'a, [f32]> {
6043    if dtype == TensorDtype::Q8_2f {
6044        x.iter().zip(col_field).map(|(a, c)| a * c).collect()
6045    } else {
6046        std::borrow::Cow::Borrowed(x)
6047    }
6048}
6049
6050/// θ col-field fold for q8_2f activations. Borrowed pass-through for
6051/// every other dtype, using thread-local buffers to eliminate per-matvec allocations.
6052pub(crate) fn prescale_with<R, F: FnOnce(&[f32]) -> R>(
6053    x: &[f32],
6054    col_field: &[f32],
6055    dtype: TensorDtype,
6056    buf_id: u8,
6057    f: F,
6058) -> R {
6059    if dtype == TensorDtype::Q8_2f {
6060        if buf_id == 1 {
6061            PRESCALE_BUF1.with(|b| {
6062                let mut buf = b.borrow_mut();
6063                buf.clear();
6064                buf.extend(x.iter().zip(col_field).map(|(a, c)| a * c));
6065                f(&buf)
6066            })
6067        } else {
6068            PRESCALE_BUF2.with(|b| {
6069                let mut buf = b.borrow_mut();
6070                buf.clear();
6071                buf.extend(x.iter().zip(col_field).map(|(a, c)| a * c));
6072                f(&buf)
6073            })
6074        }
6075    } else {
6076        f(x)
6077    }
6078}
6079
6080// ───────────────────── x86-64 AVX2 kernels (roadmap этап 2) ─────────────────────
6081
6082/// AVX2+FMA available? Default ON when the CPU supports both;
6083/// `CMF_AVX2=0` disables (falls back to the autovectorized loops).
6084#[cfg(target_arch = "x86_64")]
6085pub(crate) fn avx2_enabled() -> bool {
6086    use std::sync::OnceLock;
6087    static ON: OnceLock<bool> = OnceLock::new();
6088    *ON.get_or_init(|| {
6089        std::env::var("CMF_AVX2").map(|v| v != "0").unwrap_or(true)
6090            && std::arch::is_x86_feature_detected!("avx2")
6091            && std::arch::is_x86_feature_detected!("fma")
6092    })
6093}
6094
6095/// AVX2 A8W8 allowed? The quantized-activation contract is switched by
6096/// the SAME env as the ARM SDOT path: `CMF_SDOT=0` keeps exact kernels
6097/// (the golden-parity exact gate relies on it) — AVX2 f32 kernels stay
6098/// active either way, they are exact (regrouped sums only).
6099#[cfg(target_arch = "x86_64")]
6100fn avx2_a8w8_enabled() -> bool {
6101    use std::sync::OnceLock;
6102    static ON: OnceLock<bool> = OnceLock::new();
6103    *ON.get_or_init(|| {
6104        avx2_enabled() && std::env::var("CMF_SDOT").map(|v| v != "0").unwrap_or(true)
6105    })
6106}
6107
6108/// A8W8 quantized-activation path available on THIS machine? One
6109/// switch across architectures: ARM dotprod (CMF_SDOT) or x86 AVX2
6110/// (CMF_AVX2 + the same CMF_SDOT exact-contract override).
6111#[inline]
6112pub(crate) fn a8w8_enabled() -> bool {
6113    #[cfg(target_arch = "aarch64")]
6114    {
6115        sdot_enabled()
6116    }
6117    #[cfg(target_arch = "x86_64")]
6118    {
6119        avx2_a8w8_enabled()
6120    }
6121    #[cfg(not(any(target_arch = "aarch64", target_arch = "x86_64")))]
6122    {
6123        false
6124    }
6125}
6126
6127/// int8·int8 dot dispatch: SDOT on ARM; AVX-512 VNNI (vpdpbusd) or AVX2
6128/// maddubs on x86. Callers are gated by `a8w8_enabled()`.
6129#[inline]
6130#[allow(unreachable_code)]
6131fn dot_i8_i8(w: &[u8], xq: &[i8]) -> i32 {
6132    #[cfg(target_arch = "aarch64")]
6133    unsafe {
6134        return dot_i8_sdot(w, xq);
6135    }
6136    #[cfg(target_arch = "x86_64")]
6137    unsafe {
6138        if avx512vnni_enabled() {
6139            return dot_i8_i8_vnni(w, xq);
6140        }
6141        return dot_i8_i8_avx2(w, xq);
6142    }
6143    w.iter()
6144        .zip(xq)
6145        .map(|(&a, &b)| (a as i8) as i32 * b as i32)
6146        .sum()
6147}
6148
6149/// AVX-512 VNNI available? (F+BW+VL+VNNI; `CMF_AVX512=0` falls back to
6150/// AVX2.) VL matters: short 32-byte groups (q4/vbit) ride the 256-bit
6151/// `vpdpbusd` encoding.
6152#[cfg(target_arch = "x86_64")]
6153fn avx512vnni_enabled() -> bool {
6154    use std::sync::OnceLock;
6155    static ON: OnceLock<bool> = OnceLock::new();
6156    *ON.get_or_init(|| {
6157        std::env::var("CMF_AVX512")
6158            .map(|v| v != "0")
6159            .unwrap_or(true)
6160            && std::arch::is_x86_feature_detected!("avx512f")
6161            && std::arch::is_x86_feature_detected!("avx512bw")
6162            && std::arch::is_x86_feature_detected!("avx512vl")
6163            && std::arch::is_x86_feature_detected!("avx512vnni")
6164    })
6165}
6166
6167/// Grouped-codec VNNI arms (the q4t/q4b/q1/q1t tile kernels): default
6168/// ON where AVX-512 VNNI exists (`CMF_VNNI_TILES=0` opt-out). Measured
6169/// on Ryzen 7950X (Zen4, 3 alternating process pairs, blocked GEMM
6170/// 4864×896 b=256): q4t 63→68 GF/s (+8%), q1 53→56 (+6%), q4b 72→75
6171/// (+4%) — consistent, no leg regressed. The tile kernels keep a
6172/// horizontal reduce per 32-weight group, so the `vpdpbusd` saving is
6173/// smaller than the long-dot q8 win (+13%), but it is real and free.
6174#[cfg(target_arch = "x86_64")]
6175fn vnni_tiles_enabled() -> bool {
6176    use std::sync::OnceLock;
6177    static ON: OnceLock<bool> = OnceLock::new();
6178    *ON.get_or_init(|| {
6179        std::env::var("CMF_VNNI_TILES")
6180            .map(|v| v != "0")
6181            .unwrap_or(true)
6182            && avx512vnni_enabled()
6183    })
6184}
6185
6186/// One 256-bit u8×i8 dot → i32 via `vpdpbusd` into a fresh accumulator
6187/// plus the same horizontal reduce the AVX2 kernels use. Products are
6188/// bounded (|w| ≤ 8 or ≤ 1), so maddubs never saturated — the i32 sum
6189/// is bit-identical to the maddubs+madd pair it replaces.
6190#[cfg(target_arch = "x86_64")]
6191#[target_feature(enable = "avx2,avx512f,avx512bw,avx512vl,avx512vnni")]
6192#[inline]
6193unsafe fn dpbusd_hsum(aw: core::arch::x86_64::__m256i, xs: core::arch::x86_64::__m256i) -> i32 {
6194    // SAFETY: pure register math.
6195    unsafe {
6196        use core::arch::x86_64::*;
6197        let d = _mm256_dpbusd_epi32(_mm256_setzero_si256(), aw, xs);
6198        let hi128 = _mm256_extracti128_si256::<1>(d);
6199        let s128 = _mm_add_epi32(_mm256_castsi256_si128(d), hi128);
6200        let s64 = _mm_add_epi32(s128, _mm_srli_si128::<8>(s128));
6201        let s32 = _mm_add_epi32(s64, _mm_srli_si128::<4>(s64));
6202        _mm_cvtsi128_si32(s32)
6203    }
6204}
6205
6206/// int8·int8 via AVX-512 VNNI: `vpdpbusd` fuses the maddubs+madd+add
6207/// triple into one u8×i8 dot-accumulate. AVX-512 has no vpsignb, so the
6208/// |w|·sign(x,w) trick becomes |w| × (x negated where w<0) via a mask
6209/// subtract — w==0 lanes contribute 0 through |w|=0 either way.
6210#[cfg(target_arch = "x86_64")]
6211#[target_feature(enable = "avx2,avx512f,avx512bw,avx512vl,avx512vnni")]
6212unsafe fn dot_i8_i8_vnni(w: &[u8], xq: &[i8]) -> i32 {
6213    // SAFETY: callers uphold slice-length contracts (see call sites).
6214    unsafe {
6215        use core::arch::x86_64::*;
6216        let n = w.len();
6217        let mut j = 0usize;
6218        let mut total: i32;
6219        // 4 independent accumulators: vpdpbusd is its own loop-carried
6220        // dependency (~5-cycle latency) — a single-acc loop runs
6221        // latency-bound and LOSES to the AVX2 maddubs kernel, measured
6222        // on Granite Rapids.
6223        {
6224            #[inline(always)]
6225            unsafe fn step(
6226                w: *const u8,
6227                x: *const i8,
6228                acc: core::arch::x86_64::__m512i,
6229            ) -> core::arch::x86_64::__m512i {
6230                unsafe {
6231                    use core::arch::x86_64::*;
6232                    let wv = _mm512_loadu_si512(w as *const _);
6233                    let xv = _mm512_loadu_si512(x as *const _);
6234                    let aw = _mm512_abs_epi8(wv);
6235                    let neg = _mm512_movepi8_mask(wv);
6236                    let sx = _mm512_mask_sub_epi8(xv, neg, _mm512_setzero_si512(), xv);
6237                    _mm512_dpbusd_epi32(acc, aw, sx)
6238                }
6239            }
6240            let (mut a0, mut a1, mut a2, mut a3) = (
6241                _mm512_setzero_si512(),
6242                _mm512_setzero_si512(),
6243                _mm512_setzero_si512(),
6244                _mm512_setzero_si512(),
6245            );
6246            while j + 256 <= n {
6247                a0 = step(w.as_ptr().add(j), xq.as_ptr().add(j), a0);
6248                a1 = step(w.as_ptr().add(j + 64), xq.as_ptr().add(j + 64), a1);
6249                a2 = step(w.as_ptr().add(j + 128), xq.as_ptr().add(j + 128), a2);
6250                a3 = step(w.as_ptr().add(j + 192), xq.as_ptr().add(j + 192), a3);
6251                j += 256;
6252            }
6253            while j + 64 <= n {
6254                a0 = step(w.as_ptr().add(j), xq.as_ptr().add(j), a0);
6255                j += 64;
6256            }
6257            let s01 = _mm512_add_epi32(a0, a1);
6258            let s23 = _mm512_add_epi32(a2, a3);
6259            total = _mm512_reduce_add_epi32(_mm512_add_epi32(s01, s23));
6260        }
6261        // 32-wide (q4/vbit groups are exactly 32 bytes).
6262        if j + 32 <= n {
6263            let wv = _mm256_loadu_si256(w.as_ptr().add(j) as *const __m256i);
6264            let xv = _mm256_loadu_si256(xq.as_ptr().add(j) as *const __m256i);
6265            let d = _mm256_dpbusd_epi32(
6266                _mm256_setzero_si256(),
6267                _mm256_abs_epi8(wv),
6268                _mm256_sign_epi8(xv, wv),
6269            );
6270            let hi128 = _mm256_extracti128_si256::<1>(d);
6271            let s128 = _mm_add_epi32(_mm256_castsi256_si128(d), hi128);
6272            let s64 = _mm_add_epi32(s128, _mm_srli_si128::<8>(s128));
6273            let s32 = _mm_add_epi32(s64, _mm_srli_si128::<4>(s64));
6274            total += _mm_cvtsi128_si32(s32);
6275            j += 32;
6276        }
6277        while j < n {
6278            total += (w[j] as i8) as i32 * xq[j] as i32;
6279            j += 1;
6280        }
6281        total
6282    }
6283}
6284
6285/// i8 row · f32 x via AVX2/FMA (x86 mirror of `dot_i8_f32_neon`).
6286#[cfg(target_arch = "x86_64")]
6287#[target_feature(enable = "avx2,fma")]
6288unsafe fn dot_i8_f32_avx2(w: &[u8], x: &[f32]) -> f32 {
6289    // SAFETY: callers uphold slice-length contracts (see call sites).
6290    unsafe {
6291        use core::arch::x86_64::*;
6292        let n = x.len();
6293        let wp = w.as_ptr();
6294        let xp = x.as_ptr();
6295        let (mut a0, mut a1) = (_mm256_setzero_ps(), _mm256_setzero_ps());
6296        let mut j = 0usize;
6297        while j + 16 <= n {
6298            let wb = _mm_loadu_si128(wp.add(j) as *const __m128i);
6299            let lo = _mm256_cvtepi8_epi32(wb);
6300            let hi = _mm256_cvtepi8_epi32(_mm_srli_si128::<8>(wb));
6301            a0 = _mm256_fmadd_ps(_mm256_cvtepi32_ps(lo), _mm256_loadu_ps(xp.add(j)), a0);
6302            a1 = _mm256_fmadd_ps(_mm256_cvtepi32_ps(hi), _mm256_loadu_ps(xp.add(j + 8)), a1);
6303            j += 16;
6304        }
6305        let acc = _mm256_add_ps(a0, a1);
6306        let hi128 = _mm256_extractf128_ps::<1>(acc);
6307        let s128 = _mm_add_ps(_mm256_castps256_ps128(acc), hi128);
6308        let s64 = _mm_add_ps(s128, _mm_movehl_ps(s128, s128));
6309        let s32 = _mm_add_ss(s64, _mm_shuffle_ps::<1>(s64, s64));
6310        let mut sum = _mm_cvtss_f32(s32);
6311        while j < n {
6312            sum += (*wp.add(j) as i8) as f32 * *xp.add(j);
6313            j += 1;
6314        }
6315        sum
6316    }
6317}
6318
6319/// int8(weight)·int8(activation) → i32 via AVX2 maddubs — the x86
6320/// analogue of the SDOT A8W8 path. `maddubs` takes u8×i8, so the
6321/// standard sign trick applies: |w| × sign(x, w) ≡ w × x per lane.
6322/// Pair saturation is safe: |w|≤128, |x|≤127 → 2·128·127 < 32767.
6323#[cfg(target_arch = "x86_64")]
6324#[target_feature(enable = "avx2")]
6325unsafe fn dot_i8_i8_avx2(w: &[u8], xq: &[i8]) -> i32 {
6326    // SAFETY: callers uphold slice-length contracts (see call sites).
6327    unsafe {
6328        use core::arch::x86_64::*;
6329        let n = w.len();
6330        let ones = _mm256_set1_epi16(1);
6331        let mut acc = _mm256_setzero_si256();
6332        let mut j = 0usize;
6333        while j + 32 <= n {
6334            let wv = _mm256_loadu_si256(w.as_ptr().add(j) as *const __m256i);
6335            let xv = _mm256_loadu_si256(xq.as_ptr().add(j) as *const __m256i);
6336            let p16 = _mm256_maddubs_epi16(_mm256_abs_epi8(wv), _mm256_sign_epi8(xv, wv));
6337            acc = _mm256_add_epi32(acc, _mm256_madd_epi16(p16, ones));
6338            j += 32;
6339        }
6340        let hi128 = _mm256_extracti128_si256::<1>(acc);
6341        let s128 = _mm_add_epi32(_mm256_castsi256_si128(acc), hi128);
6342        let s64 = _mm_add_epi32(s128, _mm_srli_si128::<8>(s128));
6343        let s32 = _mm_add_epi32(s64, _mm_srli_si128::<4>(s64));
6344        let mut s = _mm_cvtsi128_si32(s32);
6345        while j < n {
6346            s += (w[j] as i8) as i32 * xq[j] as i32;
6347            j += 1;
6348        }
6349        s
6350    }
6351}
6352
6353/// smmla 2×4: one instruction covers a 2-row × 2-activation × 8-deep
6354/// tile (32 MACs vs sdot's 16) — the weight pair loads once per 8-k
6355/// slice as a combined 2×8 register and meets two activation pairs.
6356#[cfg(target_arch = "aarch64")]
6357#[target_feature(enable = "neon,i8mm")]
6358unsafe fn dot_i8_smmla_2x4(w0: &[u8], w1: &[u8], xs: [&[i8]; 4]) -> [[i32; 4]; 2] {
6359    // SAFETY: callers uphold slice-length contracts.
6360    unsafe {
6361        use core::arch::aarch64::*;
6362        use core::arch::asm;
6363        let n = w0.len();
6364        let w0p = w0.as_ptr() as *const i8;
6365        let w1p = w1.as_ptr() as *const i8;
6366        // acc01 holds [c(r0,x0) c(r0,x1) c(r1,x0) c(r1,x1)]; acc23 the
6367        // same for x2/x3.
6368        let mut acc01 = vdupq_n_s32(0);
6369        let mut acc23 = vdupq_n_s32(0);
6370        let mut i = 0usize;
6371        while i + 8 <= n {
6372            let wa = vcombine_s8(vld1_s8(w0p.add(i)), vld1_s8(w1p.add(i)));
6373            let xb01 = vcombine_s8(
6374                vld1_s8(xs[0].as_ptr().add(i)),
6375                vld1_s8(xs[1].as_ptr().add(i)),
6376            );
6377            let xb23 = vcombine_s8(
6378                vld1_s8(xs[2].as_ptr().add(i)),
6379                vld1_s8(xs[3].as_ptr().add(i)),
6380            );
6381            asm!(
6382                "smmla {a01:v}.4s, {w:v}.16b, {x01:v}.16b",
6383                "smmla {a23:v}.4s, {w:v}.16b, {x23:v}.16b",
6384                a01 = inout(vreg) acc01, a23 = inout(vreg) acc23,
6385                w = in(vreg) wa, x01 = in(vreg) xb01, x23 = in(vreg) xb23,
6386                options(pure, nomem, nostack),
6387            );
6388            i += 8;
6389        }
6390        let mut out = [[0i32; 4]; 2];
6391        let a01: [i32; 4] = core::mem::transmute(acc01);
6392        let a23: [i32; 4] = core::mem::transmute(acc23);
6393        out[0][0] = a01[0];
6394        out[0][1] = a01[1];
6395        out[1][0] = a01[2];
6396        out[1][1] = a01[3];
6397        out[0][2] = a23[0];
6398        out[0][3] = a23[1];
6399        out[1][2] = a23[2];
6400        out[1][3] = a23[3];
6401        if i < n {
6402            for (k, x) in xs.iter().enumerate() {
6403                for j in i..n {
6404                    out[0][k] += (w0[j] as i8) as i32 * x[j] as i32;
6405                    out[1][k] += (w1[j] as i8) as i32 * x[j] as i32;
6406                }
6407            }
6408        }
6409        out
6410    }
6411}
6412
6413/// ARM twin of the x86 blocked prefill GEMM: two weight rows stay in
6414/// registers across four activation streams, eight sdot accumulators.
6415/// (The per-row form re-read each W row once per activation.)
6416#[cfg(target_arch = "aarch64")]
6417#[target_feature(enable = "neon,dotprod")]
6418unsafe fn dot_i8_sdot_2x4(w0: &[u8], w1: &[u8], xs: [&[i8]; 4]) -> [[i32; 4]; 2] {
6419    // SAFETY: callers uphold slice-length contracts.
6420    unsafe {
6421        use core::arch::aarch64::*;
6422        use core::arch::asm;
6423        let n = w0.len();
6424        let w0p = w0.as_ptr() as *const i8;
6425        let w1p = w1.as_ptr() as *const i8;
6426        let mut acc = [[vdupq_n_s32(0); 4]; 2];
6427        let mut i = 0usize;
6428        while i + 16 <= n {
6429            let wv0 = vld1q_s8(w0p.add(i));
6430            let wv1 = vld1q_s8(w1p.add(i));
6431            for (k, x) in xs.iter().enumerate() {
6432                let xv = vld1q_s8(x.as_ptr().add(i));
6433                let (mut a0, mut a1) = (acc[0][k], acc[1][k]);
6434                asm!(
6435                    "sdot {a0:v}.4s, {w0:v}.16b, {x:v}.16b",
6436                    "sdot {a1:v}.4s, {w1:v}.16b, {x:v}.16b",
6437                    a0 = inout(vreg) a0, a1 = inout(vreg) a1,
6438                    w0 = in(vreg) wv0, w1 = in(vreg) wv1, x = in(vreg) xv,
6439                    options(pure, nomem, nostack),
6440                );
6441                acc[0][k] = a0;
6442                acc[1][k] = a1;
6443            }
6444            i += 16;
6445        }
6446        let mut out = [[0i32; 4]; 2];
6447        for r in 0..2 {
6448            for k in 0..4 {
6449                out[r][k] = vaddvq_s32(acc[r][k]);
6450            }
6451        }
6452        if i < n {
6453            for (k, x) in xs.iter().enumerate() {
6454                for j in i..n {
6455                    out[0][k] += (w0[j] as i8) as i32 * x[j] as i32;
6456                    out[1][k] += (w1[j] as i8) as i32 * x[j] as i32;
6457                }
6458            }
6459        }
6460        out
6461    }
6462}
6463
6464/// Blocked 2 weight rows × 4 activations for the prefill GEMM
6465/// (roadmap P0: packed panels + multi-row accumulators). The two rows'
6466/// abs() live in registers across all four activation streams; the
6467/// sign-fixup is recomputed per pair (the price of the maddubs trick).
6468/// Returns raw i8·i8 dots; the caller applies scales and outliers.
6469#[cfg(target_arch = "x86_64")]
6470#[target_feature(enable = "avx2")]
6471unsafe fn dot_i8_i8_avx2_2x4(w0: &[u8], w1: &[u8], xs: [&[i8]; 4]) -> [[i32; 4]; 2] {
6472    // SAFETY: callers uphold slice-length contracts.
6473    unsafe {
6474        use core::arch::x86_64::*;
6475        let n = w0.len();
6476        let ones = _mm256_set1_epi16(1);
6477        let mut acc = [[_mm256_setzero_si256(); 4]; 2];
6478        let mut j = 0usize;
6479        while j + 32 <= n {
6480            let wv0 = _mm256_loadu_si256(w0.as_ptr().add(j) as *const __m256i);
6481            let wv1 = _mm256_loadu_si256(w1.as_ptr().add(j) as *const __m256i);
6482            let aw0 = _mm256_abs_epi8(wv0);
6483            let aw1 = _mm256_abs_epi8(wv1);
6484            for (k, x) in xs.iter().enumerate() {
6485                let xv = _mm256_loadu_si256(x.as_ptr().add(j) as *const __m256i);
6486                let p0 = _mm256_maddubs_epi16(aw0, _mm256_sign_epi8(xv, wv0));
6487                acc[0][k] = _mm256_add_epi32(acc[0][k], _mm256_madd_epi16(p0, ones));
6488                let p1 = _mm256_maddubs_epi16(aw1, _mm256_sign_epi8(xv, wv1));
6489                acc[1][k] = _mm256_add_epi32(acc[1][k], _mm256_madd_epi16(p1, ones));
6490            }
6491            j += 32;
6492        }
6493        let mut out = [[0i32; 4]; 2];
6494        for r in 0..2 {
6495            for k in 0..4 {
6496                let a = acc[r][k];
6497                let hi128 = _mm256_extracti128_si256::<1>(a);
6498                let s128 = _mm_add_epi32(_mm256_castsi256_si128(a), hi128);
6499                let s64 = _mm_add_epi32(s128, _mm_srli_si128::<8>(s128));
6500                let s32 = _mm_add_epi32(s64, _mm_srli_si128::<4>(s64));
6501                out[r][k] = _mm_cvtsi128_si32(s32);
6502            }
6503        }
6504        if j < n {
6505            for (k, x) in xs.iter().enumerate() {
6506                for i in j..n {
6507                    out[0][k] += (w0[i] as i8) as i32 * x[i] as i32;
6508                    out[1][k] += (w1[i] as i8) as i32 * x[i] as i32;
6509                }
6510            }
6511        }
6512        out
6513    }
6514}
6515
6516/// AVX2/VNNI q8 row dot with exact outlier correction (x86 mirror of
6517/// `row_dot_sdot` — same A8W8 contract). With AVX-512 VNNI the row goes
6518/// through the bias trick: Σ(w+128)·x via pure `vpdpbusd` (no per-lane
6519/// sign fixups), corrected by −128·Σx with Σx precomputed per split.
6520#[cfg(target_arch = "x86_64")]
6521#[inline]
6522fn row_dot_avx2(row: &[u8], act: &SplitAct) -> f32 {
6523    let dot = if avx512vnni_enabled() && row.len() >= 64 {
6524        (unsafe { dot_u8p128_i8_vnni(row, &act.xq) }) - 128 * act.xsum
6525    } else {
6526        unsafe { dot_i8_i8_avx2(row, &act.xq) }
6527    };
6528    let mut acc = dot as f32 * act.sx;
6529    for &(j, xv) in &act.outliers {
6530        acc += (row[j] as i8) as f32 * xv;
6531    }
6532    acc
6533}
6534
6535/// Σ (w[i]+128)·x[i] via pure `vpdpbusd` — the caller subtracts
6536/// 128·Σx. Four independent accumulators (dpbusd is ~5-cycle latency;
6537/// a single-acc loop runs latency-bound, measured on Granite Rapids).
6538#[cfg(target_arch = "x86_64")]
6539#[target_feature(enable = "avx2,avx512f,avx512bw,avx512vl,avx512vnni")]
6540unsafe fn dot_u8p128_i8_vnni(w: &[u8], xq: &[i8]) -> i32 {
6541    // SAFETY: callers uphold slice-length contracts (see call sites).
6542    unsafe {
6543        use core::arch::x86_64::*;
6544        let n = w.len();
6545        let flip = _mm512_set1_epi8(-128); // XOR 0x80: i8 w → u8 (w+128)
6546        #[inline(always)]
6547        unsafe fn step(
6548            w: *const u8,
6549            x: *const i8,
6550            flip: core::arch::x86_64::__m512i,
6551            acc: core::arch::x86_64::__m512i,
6552        ) -> core::arch::x86_64::__m512i {
6553            unsafe {
6554                use core::arch::x86_64::*;
6555                let wv = _mm512_xor_si512(_mm512_loadu_si512(w as *const _), flip);
6556                _mm512_dpbusd_epi32(acc, wv, _mm512_loadu_si512(x as *const _))
6557            }
6558        }
6559        let (mut a0, mut a1, mut a2, mut a3) = (
6560            _mm512_setzero_si512(),
6561            _mm512_setzero_si512(),
6562            _mm512_setzero_si512(),
6563            _mm512_setzero_si512(),
6564        );
6565        let mut j = 0usize;
6566        while j + 256 <= n {
6567            a0 = step(w.as_ptr().add(j), xq.as_ptr().add(j), flip, a0);
6568            a1 = step(w.as_ptr().add(j + 64), xq.as_ptr().add(j + 64), flip, a1);
6569            a2 = step(w.as_ptr().add(j + 128), xq.as_ptr().add(j + 128), flip, a2);
6570            a3 = step(w.as_ptr().add(j + 192), xq.as_ptr().add(j + 192), flip, a3);
6571            j += 256;
6572        }
6573        while j + 64 <= n {
6574            a0 = step(w.as_ptr().add(j), xq.as_ptr().add(j), flip, a0);
6575            j += 64;
6576        }
6577        let mut total = _mm512_reduce_add_epi32(_mm512_add_epi32(
6578            _mm512_add_epi32(a0, a1),
6579            _mm512_add_epi32(a2, a3),
6580        ));
6581        // Scalar tail: (w as i8) + 128 ≡ (w as u8) ^ 0x80.
6582        while j < n {
6583            total += ((w[j] ^ 0x80) as i32) * xq[j] as i32;
6584            j += 1;
6585        }
6586        total
6587    }
6588}
6589
6590/// One q4 row via AVX2: nibbles → centered i8 (unpacklo/hi restores the
6591/// writer's flat order, same as the NEON vzip pair), maddubs against
6592/// the pre-quantized activation group, × the group's f16 scale. Pair
6593/// saturation safe: |w|≤8, |x|≤127 → 2·8·127 ≪ 32767. Mirror of
6594/// `dot_q4_row_sdot`.
6595#[cfg(target_arch = "x86_64")]
6596#[target_feature(enable = "avx2")]
6597unsafe fn dot_q4_row_avx2(packed: &[u8], scales: &[u8], g0: usize, gpr: usize, xq: &[i8]) -> f32 {
6598    // SAFETY: callers uphold slice-length contracts (16 packed bytes and
6599    // 2 scale bytes per group; xq.len() == gpr·GROUP_SIZE).
6600    unsafe {
6601        use core::arch::x86_64::*;
6602        let lomask = _mm_set1_epi8(0x0F);
6603        let eight = _mm256_set1_epi8(8);
6604        let ones = _mm256_set1_epi16(1);
6605        let mut acc = 0f32;
6606        for gi in 0..gpr {
6607            let g = g0 + gi;
6608            let s = f16_to_f32(u16::from_le_bytes([scales[g * 2], scales[g * 2 + 1]]));
6609            let b = _mm_loadu_si128(packed.as_ptr().add(g * 16) as *const __m128i);
6610            let lo = _mm_and_si128(b, lomask);
6611            let hi = _mm_and_si128(_mm_srli_epi16::<4>(b), lomask);
6612            let w = _mm256_sub_epi8(
6613                _mm256_set_m128i(_mm_unpackhi_epi8(lo, hi), _mm_unpacklo_epi8(lo, hi)),
6614                eight,
6615            );
6616            let x = _mm256_loadu_si256(xq.as_ptr().add(gi * GROUP_SIZE) as *const __m256i);
6617            let p16 = _mm256_maddubs_epi16(_mm256_abs_epi8(w), _mm256_sign_epi8(x, w));
6618            let d = _mm256_madd_epi16(p16, ones);
6619            let hi128 = _mm256_extracti128_si256::<1>(d);
6620            let s128 = _mm_add_epi32(_mm256_castsi256_si128(d), hi128);
6621            let s64 = _mm_add_epi32(s128, _mm_srli_si128::<8>(s128));
6622            let s32 = _mm_add_epi32(s64, _mm_srli_si128::<4>(s64));
6623            acc += _mm_cvtsi128_si32(s32) as f32 * s;
6624        }
6625        acc
6626    }
6627}
6628
6629/// Two-activation q4 row via AVX2: nibbles unpacked ONCE per group,
6630/// both activations dotted against the same centered i8 register.
6631#[cfg(target_arch = "x86_64")]
6632#[target_feature(enable = "avx2")]
6633unsafe fn dot_q4_row_avx2_2(
6634    packed: &[u8],
6635    scales: &[u8],
6636    g0: usize,
6637    gpr: usize,
6638    xq1: &[i8],
6639    xq2: &[i8],
6640) -> (f32, f32) {
6641    // SAFETY: callers uphold slice-length contracts (see dot_q4_row_avx2).
6642    unsafe {
6643        use core::arch::x86_64::*;
6644        let lomask = _mm_set1_epi8(0x0F);
6645        let eight = _mm256_set1_epi8(8);
6646        let ones = _mm256_set1_epi16(1);
6647        let (mut acc1, mut acc2) = (0f32, 0f32);
6648        #[inline(always)]
6649        unsafe fn hsum(d: core::arch::x86_64::__m256i) -> i32 {
6650            unsafe {
6651                use core::arch::x86_64::*;
6652                let hi128 = _mm256_extracti128_si256::<1>(d);
6653                let s128 = _mm_add_epi32(_mm256_castsi256_si128(d), hi128);
6654                let s64 = _mm_add_epi32(s128, _mm_srli_si128::<8>(s128));
6655                let s32 = _mm_add_epi32(s64, _mm_srli_si128::<4>(s64));
6656                _mm_cvtsi128_si32(s32)
6657            }
6658        }
6659        for gi in 0..gpr {
6660            let g = g0 + gi;
6661            let s = f16_to_f32(u16::from_le_bytes([scales[g * 2], scales[g * 2 + 1]]));
6662            let b = _mm_loadu_si128(packed.as_ptr().add(g * 16) as *const __m128i);
6663            let lo = _mm_and_si128(b, lomask);
6664            let hi = _mm_and_si128(_mm_srli_epi16::<4>(b), lomask);
6665            let w = _mm256_sub_epi8(
6666                _mm256_set_m128i(_mm_unpackhi_epi8(lo, hi), _mm_unpacklo_epi8(lo, hi)),
6667                eight,
6668            );
6669            let aw = _mm256_abs_epi8(w);
6670            let x1 = _mm256_loadu_si256(xq1.as_ptr().add(gi * GROUP_SIZE) as *const __m256i);
6671            let x2 = _mm256_loadu_si256(xq2.as_ptr().add(gi * GROUP_SIZE) as *const __m256i);
6672            let d1 = _mm256_madd_epi16(_mm256_maddubs_epi16(aw, _mm256_sign_epi8(x1, w)), ones);
6673            let d2 = _mm256_madd_epi16(_mm256_maddubs_epi16(aw, _mm256_sign_epi8(x2, w)), ones);
6674            acc1 += hsum(d1) as f32 * s;
6675            acc2 += hsum(d2) as f32 * s;
6676        }
6677        (acc1, acc2)
6678    }
6679}
6680
6681/// One q8 row range via AVX2 (x86 mirror of `q8_range_sdot`).
6682#[cfg(target_arch = "x86_64")]
6683fn q8_range_avx2(
6684    q: &[u8],
6685    row_scale: &[f32],
6686    act: &SplitAct,
6687    cols: usize,
6688    out_addr: SendMut,
6689    start: usize,
6690    end: usize,
6691) {
6692    for o in start..end {
6693        let v = row_dot_avx2(&q[o * cols..(o + 1) * cols], act) * row_scale[o];
6694        // SAFETY: disjoint row ranges per worker.
6695        unsafe { *out_addr.at(o) = v };
6696    }
6697}
6698
6699/// Two-input q8 row range via AVX2 (x86 mirror of `q8_range2_sdot`).
6700#[cfg(target_arch = "x86_64")]
6701#[allow(clippy::too_many_arguments)]
6702fn q8_range2_avx2(
6703    q: &[u8],
6704    row_scale: &[f32],
6705    a1: &SplitAct,
6706    a2: &SplitAct,
6707    cols: usize,
6708    p1: SendMut,
6709    p2: SendMut,
6710    start: usize,
6711    end: usize,
6712) {
6713    for o in start..end {
6714        let row = &q[o * cols..(o + 1) * cols];
6715        // SAFETY: disjoint row ranges per worker.
6716        unsafe {
6717            *p1.at(o) = row_dot_avx2(row, a1) * row_scale[o];
6718            *p2.at(o) = row_dot_avx2(row, a2) * row_scale[o];
6719        }
6720    }
6721}
6722
6723// ───────────────────── A8W8 SDOT path (port of vmfcore, ×1.78 decode) ─────────────────────
6724
6725/// ARMv8.6 i8mm (smmla): 32 int8 MACs per instruction vs sdot's 16 —
6726/// yet MEASURED 2.4× SLOWER than the blocked sdot on Apple silicon
6727/// (108 vs 264 GF/s): the on-the-fly vcombine packing and the two-
6728/// accumulator dependency chain swamp the MAC advantage, and Apple's
6729/// four SIMD pipes already keep sdot fed. OPT-IN (CMF_I8MM=1) for
6730/// field trials on Cortex-A710/X-class parts with two pipes, where the
6731/// balance may differ; a pre-interleaved weight layout (repack infra)
6732/// is the known path if it ever earns its keep.
6733#[cfg(target_arch = "aarch64")]
6734fn i8mm_enabled() -> bool {
6735    static ON: std::sync::OnceLock<bool> = std::sync::OnceLock::new();
6736    *ON.get_or_init(|| {
6737        std::env::var("CMF_I8MM").map(|v| v == "1").unwrap_or(false)
6738            && std::arch::is_aarch64_feature_detected!("i8mm")
6739    })
6740}
6741
6742/// SDOT enabled? Default ON when the CPU has ARMv8.2 dotprod;
6743/// `CMF_SDOT=0` disables (falls back to i8×f32 NEON).
6744/// (On non-ARM release builds only the test tolerance switch calls it.)
6745#[cfg_attr(not(target_arch = "aarch64"), allow(dead_code))]
6746fn sdot_enabled() -> bool {
6747    use std::sync::OnceLock;
6748    static ON: OnceLock<bool> = OnceLock::new();
6749    *ON.get_or_init(|| {
6750        let want = std::env::var("CMF_SDOT").map(|v| v != "0").unwrap_or(true);
6751        if !want {
6752            return false;
6753        }
6754
6755        #[cfg(target_arch = "aarch64")]
6756        {
6757            if std::arch::is_aarch64_feature_detected!("dotprod") {
6758                return true;
6759            }
6760            #[cfg(target_os = "android")]
6761            {
6762                if let Ok(cpuinfo) = std::fs::read_to_string("/proc/cpuinfo") {
6763                    if cpuinfo.lines().any(|l| {
6764                        (l.starts_with("Features") || l.starts_with("features"))
6765                            && l.contains("asimddp")
6766                    }) {
6767                        return true;
6768                    }
6769                }
6770            }
6771            false
6772        }
6773        #[cfg(not(target_arch = "aarch64"))]
6774        {
6775            false
6776        }
6777    })
6778}
6779
6780/// Two-field activation split (≡ vmfcore `q8_split_prep`): outlier
6781/// channels (>8·rms) are computed exactly in f32; the bulk (outliers
6782/// zeroed → clean absmax) goes through int8 SDOT. Computed ONCE per
6783/// matvec, shared by all rows/workers.
6784struct SplitAct {
6785    xq: Vec<i8>,
6786    sx: f32,
6787    outliers: Vec<(usize, f32)>,
6788    /// Σ xq — the VNNI bias-trick correction (`(w+128)·x` sums need
6789    /// `−128·Σx`); one i32 per split, computed once per matvec.
6790    #[cfg_attr(not(target_arch = "x86_64"), allow(dead_code))]
6791    xsum: i32,
6792}
6793
6794thread_local! {
6795    /// Recycled xq buffers: split_act runs for every matvec (~200/token)
6796    /// and its hidden-size allocation was steady-state heap churn.
6797    static XQ_FREE: std::cell::RefCell<Vec<Vec<i8>>> =
6798        const { std::cell::RefCell::new(Vec::new()) };
6799}
6800
6801impl Drop for SplitAct {
6802    fn drop(&mut self) {
6803        let buf = std::mem::take(&mut self.xq);
6804        if buf.capacity() > 0 {
6805            XQ_FREE.with(|f| {
6806                let mut f = f.borrow_mut();
6807                if f.len() < 16 {
6808                    f.push(buf);
6809                }
6810            });
6811        }
6812    }
6813}
6814
6815fn split_act(x: &[f32]) -> SplitAct {
6816    let n = x.len();
6817    let rms = (x.iter().map(|&v| (v * v) as f64).sum::<f64>() / n.max(1) as f64).sqrt() as f32;
6818    let thr = 8.0 * rms;
6819    // One pass: collect outliers and the bulk absmax (outliers excluded —
6820    // identical to the old zero-then-fold over a copied buffer, minus the
6821    // full-vector copy).
6822    let mut outliers: Vec<(usize, f32)> = Vec::new();
6823    let mut amax = 0f32;
6824    for (j, &v) in x.iter().enumerate() {
6825        let a = v.abs();
6826        if a > thr {
6827            outliers.push((j, v));
6828        } else if a > amax {
6829            amax = a;
6830        }
6831    }
6832    let sx = if amax > 0.0 { amax / 127.0 } else { 1.0 };
6833    let inv = 1.0 / sx;
6834    let mut xq = XQ_FREE.with(|f| f.borrow_mut().pop()).unwrap_or_default();
6835    xq.clear();
6836    xq.reserve(n);
6837    if outliers.is_empty() {
6838        xq.extend(
6839            x.iter()
6840                .map(|&v| (v * inv).round().clamp(-127.0, 127.0) as i8),
6841        );
6842    } else {
6843        // Outlier slots quantize to 0 (their exact term is added later).
6844        xq.extend(x.iter().map(|&v| {
6845            if v.abs() > thr {
6846                0
6847            } else {
6848                (v * inv).round().clamp(-127.0, 127.0) as i8
6849            }
6850        }));
6851    }
6852    let xsum = xq.iter().map(|&v| v as i32).sum();
6853    SplitAct {
6854        xq,
6855        sx,
6856        outliers,
6857        xsum,
6858    }
6859}
6860
6861fn split_act_q8_2f(x: &[f32], col: &[f32]) -> SplitAct {
6862    let n = x.len();
6863    let rms = (x
6864        .iter()
6865        .zip(col)
6866        .map(|(&a, &c)| {
6867            let v = a * c;
6868            (v * v) as f64
6869        })
6870        .sum::<f64>()
6871        / n.max(1) as f64)
6872        .sqrt() as f32;
6873    let thr = 8.0 * rms;
6874
6875    let mut outliers = Vec::new();
6876    let mut amax = 0f32;
6877    for (j, (&a, &c)) in x.iter().zip(col).enumerate() {
6878        let v = a * c;
6879        let s = v.abs();
6880        if s > thr {
6881            outliers.push((j, v));
6882        } else if s > amax {
6883            amax = s;
6884        }
6885    }
6886
6887    let sx = if amax > 0.0 { amax / 127.0 } else { 1.0 };
6888    let inv = 1.0 / sx;
6889    let mut xq = XQ_FREE.with(|f| f.borrow_mut().pop()).unwrap_or_default();
6890    xq.clear();
6891    xq.reserve(n);
6892    if outliers.is_empty() {
6893        xq.extend(
6894            x.iter()
6895                .zip(col)
6896                .map(|(&a, &c)| ((a * c) * inv).round().clamp(-127.0, 127.0) as i8),
6897        );
6898    } else {
6899        xq.extend(x.iter().zip(col).map(|(&a, &c)| {
6900            let v = a * c;
6901            if v.abs() > thr {
6902                0
6903            } else {
6904                (v * inv).round().clamp(-127.0, 127.0) as i8
6905            }
6906        }));
6907    }
6908    let xsum = xq.iter().map(|&v| v as i32).sum();
6909    SplitAct {
6910        xq,
6911        sx,
6912        outliers,
6913        xsum,
6914    }
6915}
6916
6917/// int8(weight)·int8(activation) → i32 via `sdot` (inline asm — the
6918/// vdotq intrinsic is unstable; port of vmfcore `dot_i8_sdot`).
6919#[cfg(target_arch = "aarch64")]
6920#[target_feature(enable = "neon,dotprod")]
6921unsafe fn dot_i8_sdot(w: &[u8], xq: &[i8]) -> i32 {
6922    // SAFETY: callers uphold slice-length contracts (see call sites).
6923    unsafe {
6924        use core::arch::aarch64::*;
6925        use core::arch::asm;
6926        let wp = w.as_ptr() as *const i8;
6927        let n = w.len();
6928        let (mut a0, mut a1, mut a2, mut a3) = (
6929            vdupq_n_s32(0),
6930            vdupq_n_s32(0),
6931            vdupq_n_s32(0),
6932            vdupq_n_s32(0),
6933        );
6934        let mut i = 0;
6935        while i + 64 <= n {
6936            let (w0, x0) = (vld1q_s8(wp.add(i)), vld1q_s8(xq.as_ptr().add(i)));
6937            let (w1, x1) = (vld1q_s8(wp.add(i + 16)), vld1q_s8(xq.as_ptr().add(i + 16)));
6938            let (w2, x2) = (vld1q_s8(wp.add(i + 32)), vld1q_s8(xq.as_ptr().add(i + 32)));
6939            let (w3, x3) = (vld1q_s8(wp.add(i + 48)), vld1q_s8(xq.as_ptr().add(i + 48)));
6940            asm!(
6941                "sdot {a0:v}.4s, {w0:v}.16b, {x0:v}.16b",
6942                "sdot {a1:v}.4s, {w1:v}.16b, {x1:v}.16b",
6943                "sdot {a2:v}.4s, {w2:v}.16b, {x2:v}.16b",
6944                "sdot {a3:v}.4s, {w3:v}.16b, {x3:v}.16b",
6945                a0 = inout(vreg) a0, a1 = inout(vreg) a1, a2 = inout(vreg) a2, a3 = inout(vreg) a3,
6946                w0 = in(vreg) w0, x0 = in(vreg) x0, w1 = in(vreg) w1, x1 = in(vreg) x1,
6947                w2 = in(vreg) w2, x2 = in(vreg) x2, w3 = in(vreg) w3, x3 = in(vreg) x3,
6948                options(pure, nomem, nostack),
6949            );
6950            i += 64;
6951        }
6952        while i + 16 <= n {
6953            let (wv, xv) = (vld1q_s8(wp.add(i)), vld1q_s8(xq.as_ptr().add(i)));
6954            asm!("sdot {a:v}.4s, {w:v}.16b, {x:v}.16b",
6955                 a = inout(vreg) a0, w = in(vreg) wv, x = in(vreg) xv, options(pure, nomem, nostack));
6956            i += 16;
6957        }
6958        let mut s = vaddvq_s32(vaddq_s32(vaddq_s32(a0, a1), vaddq_s32(a2, a3)));
6959        while i < n {
6960            s += (*wp.add(i)) as i32 * xq[i] as i32;
6961            i += 1;
6962        }
6963        s
6964    }
6965}
6966
6967/// Row-blocked SDOT: 4 output rows per pass — the activation chunk is
6968/// loaded once and reused, 4 independent accumulators hide sdot latency
6969/// (port of vmfcore `dot_i8_sdot_4rows`).
6970#[cfg(target_arch = "aarch64")]
6971#[target_feature(enable = "neon,dotprod")]
6972unsafe fn dot_i8_sdot_4rows(w0: &[u8], w1: &[u8], w2: &[u8], w3: &[u8], xq: &[i8]) -> [i32; 4] {
6973    // SAFETY: callers uphold slice-length contracts (see call sites).
6974    unsafe {
6975        use core::arch::aarch64::*;
6976        use core::arch::asm;
6977        let n = xq.len();
6978        let px = xq.as_ptr();
6979        let (p0, p1, p2, p3) = (
6980            w0.as_ptr() as *const i8,
6981            w1.as_ptr() as *const i8,
6982            w2.as_ptr() as *const i8,
6983            w3.as_ptr() as *const i8,
6984        );
6985        let (mut a0, mut a1, mut a2, mut a3) = (
6986            vdupq_n_s32(0),
6987            vdupq_n_s32(0),
6988            vdupq_n_s32(0),
6989            vdupq_n_s32(0),
6990        );
6991        let mut i = 0;
6992        while i + 16 <= n {
6993            let x = vld1q_s8(px.add(i));
6994            let v0 = vld1q_s8(p0.add(i));
6995            let v1 = vld1q_s8(p1.add(i));
6996            let v2 = vld1q_s8(p2.add(i));
6997            let v3 = vld1q_s8(p3.add(i));
6998            asm!(
6999                "sdot {a0:v}.4s, {v0:v}.16b, {x:v}.16b",
7000                "sdot {a1:v}.4s, {v1:v}.16b, {x:v}.16b",
7001                "sdot {a2:v}.4s, {v2:v}.16b, {x:v}.16b",
7002                "sdot {a3:v}.4s, {v3:v}.16b, {x:v}.16b",
7003                a0 = inout(vreg) a0, a1 = inout(vreg) a1, a2 = inout(vreg) a2, a3 = inout(vreg) a3,
7004                v0 = in(vreg) v0, v1 = in(vreg) v1, v2 = in(vreg) v2, v3 = in(vreg) v3, x = in(vreg) x,
7005                options(pure, nomem, nostack),
7006            );
7007            i += 16;
7008        }
7009        let mut r = [
7010            vaddvq_s32(a0),
7011            vaddvq_s32(a1),
7012            vaddvq_s32(a2),
7013            vaddvq_s32(a3),
7014        ];
7015        while i < n {
7016            let xi = *px.add(i) as i32;
7017            r[0] += (*p0.add(i)) as i32 * xi;
7018            r[1] += (*p1.add(i)) as i32 * xi;
7019            r[2] += (*p2.add(i)) as i32 * xi;
7020            r[3] += (*p3.add(i)) as i32 * xi;
7021            i += 1;
7022        }
7023        r
7024    }
7025}
7026
7027/// 4 interleaved rows in one pass: the repacked group is [r0[c], r1[c],
7028/// r2[c], r3[c]] per 16-byte chunk, so each iteration reads ONE 64-byte
7029/// line plus the shared activation chunk — a single sequential weight
7030/// stream per worker. Per-row accumulation is the same one-accumulator
7031/// scheme as `dot_i8_sdot_4rows`; integer sums are exact, so outputs
7032/// are bit-identical to the mmap-layout kernel.
7033#[cfg(target_arch = "aarch64")]
7034#[target_feature(enable = "neon,dotprod")]
7035unsafe fn dot_i8_sdot_4rows_il(g: &[u8], xq: &[i8]) -> [i32; 4] {
7036    // SAFETY: callers uphold slice-length contracts (g.len() == 4·n,
7037    // n % 16 == 0 — guaranteed by the repack gate).
7038    unsafe {
7039        use core::arch::aarch64::*;
7040        use core::arch::asm;
7041        let n = xq.len();
7042        let px = xq.as_ptr();
7043        let pg = g.as_ptr() as *const i8;
7044        let (mut a0, mut a1, mut a2, mut a3) = (
7045            vdupq_n_s32(0),
7046            vdupq_n_s32(0),
7047            vdupq_n_s32(0),
7048            vdupq_n_s32(0),
7049        );
7050        let mut i = 0;
7051        while i + 16 <= n {
7052            let x = vld1q_s8(px.add(i));
7053            let base = pg.add(4 * i);
7054            let v0 = vld1q_s8(base);
7055            let v1 = vld1q_s8(base.add(16));
7056            let v2 = vld1q_s8(base.add(32));
7057            let v3 = vld1q_s8(base.add(48));
7058            asm!(
7059                "sdot {a0:v}.4s, {v0:v}.16b, {x:v}.16b",
7060                "sdot {a1:v}.4s, {v1:v}.16b, {x:v}.16b",
7061                "sdot {a2:v}.4s, {v2:v}.16b, {x:v}.16b",
7062                "sdot {a3:v}.4s, {v3:v}.16b, {x:v}.16b",
7063                a0 = inout(vreg) a0, a1 = inout(vreg) a1, a2 = inout(vreg) a2, a3 = inout(vreg) a3,
7064                v0 = in(vreg) v0, v1 = in(vreg) v1, v2 = in(vreg) v2, v3 = in(vreg) v3, x = in(vreg) x,
7065                options(pure, nomem, nostack),
7066            );
7067            i += 16;
7068        }
7069        [
7070            vaddvq_s32(a0),
7071            vaddvq_s32(a1),
7072            vaddvq_s32(a2),
7073            vaddvq_s32(a3),
7074        ]
7075    }
7076}
7077
7078/// One q8 row range via SDOT (4-row blocks + tail) — the body of
7079/// `qmatvec`'s hot loop, extracted so multi-matrix jobs can drive the
7080/// SAME kernel for several tensors under one pool dispatch. `rep` — the
7081/// load-time interleaved repack (empty = mmap layout only); rows outside
7082/// full 4-row groups always come from the mmap layout.
7083#[cfg(target_arch = "aarch64")]
7084fn q8_range_sdot(
7085    q: &[u8],
7086    rep: &[u8],
7087    row_scale: &[f32],
7088    act: &SplitAct,
7089    cols: usize,
7090    out_addr: SendMut,
7091    start: usize,
7092    end: usize,
7093) {
7094    let mut o = start;
7095    // Leading rows to the group boundary (repack path only): the pool
7096    // splits row ranges arbitrarily, groups are absolute.
7097    if !rep.is_empty() {
7098        while o < end && o % 4 != 0 {
7099            let v = row_dot_sdot(&q[o * cols..(o + 1) * cols], act) * row_scale[o];
7100            unsafe { *out_addr.at(o) = v };
7101            o += 1;
7102        }
7103    }
7104    while o + 4 <= end {
7105        let r = if rep.is_empty() {
7106            unsafe {
7107                dot_i8_sdot_4rows(
7108                    &q[o * cols..(o + 1) * cols],
7109                    &q[(o + 1) * cols..(o + 2) * cols],
7110                    &q[(o + 2) * cols..(o + 3) * cols],
7111                    &q[(o + 3) * cols..(o + 4) * cols],
7112                    &act.xq,
7113                )
7114            }
7115        } else {
7116            unsafe { dot_i8_sdot_4rows_il(&rep[o * cols..(o + 4) * cols], &act.xq) }
7117        };
7118        for k in 0..4 {
7119            let mut acc = r[k] as f32 * act.sx;
7120            for &(j, xv) in &act.outliers {
7121                acc += (q[(o + k) * cols + j] as i8) as f32 * xv;
7122            }
7123            // SAFETY: disjoint row ranges per worker.
7124            unsafe { *out_addr.at(o + k) = acc * row_scale[o + k] };
7125        }
7126        o += 4;
7127    }
7128    while o < end {
7129        let v = row_dot_sdot(&q[o * cols..(o + 1) * cols], act) * row_scale[o];
7130        unsafe { *out_addr.at(o) = v };
7131        o += 1;
7132    }
7133}
7134
7135/// Two-input q8 row range via SDOT — `qmatvec2`'s hot loop, extracted
7136/// for the fused pair multi-matrix job (`matvec2_many`).
7137#[cfg(target_arch = "aarch64")]
7138#[allow(clippy::too_many_arguments)]
7139fn q8_range2_sdot(
7140    q: &[u8],
7141    row_scale: &[f32],
7142    a1: &SplitAct,
7143    a2: &SplitAct,
7144    cols: usize,
7145    p1: SendMut,
7146    p2: SendMut,
7147    start: usize,
7148    end: usize,
7149) {
7150    for o in start..end {
7151        let row = &q[o * cols..(o + 1) * cols];
7152        // SAFETY: disjoint row ranges per worker.
7153        unsafe {
7154            *p1.at(o) = row_dot_sdot(row, a1) * row_scale[o];
7155            *p2.at(o) = row_dot_sdot(row, a2) * row_scale[o];
7156        }
7157    }
7158}
7159
7160/// Two-input q8 row range, f32 kernel (non-SDOT) — same extraction.
7161#[allow(clippy::too_many_arguments)]
7162fn q8_range2_f32(
7163    q: &[u8],
7164    row_scale: &[f32],
7165    x1: &[f32],
7166    x2: &[f32],
7167    cols: usize,
7168    p1: SendMut,
7169    p2: SendMut,
7170    start: usize,
7171    end: usize,
7172) {
7173    for o in start..end {
7174        let row = &q[o * cols..(o + 1) * cols];
7175        // SAFETY: disjoint row ranges per worker.
7176        unsafe {
7177            *p1.at(o) = dot_i8_f32(row, x1) * row_scale[o];
7178            *p2.at(o) = dot_i8_f32(row, x2) * row_scale[o];
7179        }
7180    }
7181}
7182
7183/// Scalar/NEON-f32 q8 row range (non-SDOT platforms) — same extraction.
7184fn q8_range_f32(
7185    q: &[u8],
7186    row_scale: &[f32],
7187    xs: &[f32],
7188    cols: usize,
7189    out_addr: SendMut,
7190    start: usize,
7191    end: usize,
7192) {
7193    for o in start..end {
7194        let v = dot_i8_f32(&q[o * cols..(o + 1) * cols], xs) * row_scale[o];
7195        // SAFETY: disjoint row ranges per worker.
7196        unsafe { *out_addr.at(o) = v };
7197    }
7198}
7199
7200/// SDOT row dot with exact outlier correction:
7201/// `dot = sdot(w, xq)·sx + Σ_outl w[j]·x[j]` (then × row_scale by caller).
7202#[cfg(target_arch = "aarch64")]
7203#[inline]
7204fn row_dot_sdot(row: &[u8], act: &SplitAct) -> f32 {
7205    let mut acc = unsafe { dot_i8_sdot(row, &act.xq) } as f32 * act.sx;
7206    for &(j, xv) in &act.outliers {
7207        acc += (row[j] as i8) as f32 * xv;
7208    }
7209    acc
7210}
7211
7212/// One q4 row via SDOT: each 32-group's nibbles unpack to centered i8
7213/// (nib−8 ∈ [−8,7]), int8×int8 `sdot` against the pre-quantized
7214/// activation group, × the group's f16 scale. Returns Σ_g dot_g·s_g;
7215/// the caller multiplies by the activation scale and adds the exact
7216/// outlier terms (port of vmfcore `dot_q4_block_sdot`, +23% measured).
7217/// Nibble order matches the writer: element 2k = low nibble, 2k+1 = high
7218/// → zip(lo,hi) restores flat order.
7219#[cfg(target_arch = "aarch64")]
7220#[target_feature(enable = "neon,dotprod")]
7221unsafe fn dot_q4_row_sdot(packed: &[u8], scales: &[u8], g0: usize, gpr: usize, xq: &[i8]) -> f32 {
7222    // SAFETY: callers uphold slice-length contracts (16 packed bytes and
7223    // 2 scale bytes per group; xq.len() == gpr·GROUP_SIZE).
7224    unsafe {
7225        use core::arch::aarch64::*;
7226        use core::arch::asm;
7227        let lomask = vdupq_n_u8(0x0F);
7228        let eight = vdupq_n_s8(8);
7229        let mut acc = 0f32;
7230        for gi in 0..gpr {
7231            let g = g0 + gi;
7232            let s = f16_to_f32(u16::from_le_bytes([scales[g * 2], scales[g * 2 + 1]]));
7233            let b = vld1q_u8(packed.as_ptr().add(g * 16));
7234            let lo = vandq_u8(b, lomask);
7235            let hi = vshrq_n_u8::<4>(b);
7236            let e0 = vsubq_s8(vreinterpretq_s8_u8(vzip1q_u8(lo, hi)), eight);
7237            let e1 = vsubq_s8(vreinterpretq_s8_u8(vzip2q_u8(lo, hi)), eight);
7238            let x0 = vld1q_s8(xq.as_ptr().add(gi * GROUP_SIZE));
7239            let x1 = vld1q_s8(xq.as_ptr().add(gi * GROUP_SIZE + 16));
7240            let (mut a0, mut a1) = (vdupq_n_s32(0), vdupq_n_s32(0));
7241            asm!(
7242                "sdot {a0:v}.4s, {e0:v}.16b, {x0:v}.16b",
7243                "sdot {a1:v}.4s, {e1:v}.16b, {x1:v}.16b",
7244                a0 = inout(vreg) a0, a1 = inout(vreg) a1,
7245                e0 = in(vreg) e0, x0 = in(vreg) x0, e1 = in(vreg) e1, x1 = in(vreg) x1,
7246                options(pure, nomem, nostack),
7247            );
7248            acc += vaddvq_s32(vaddq_s32(a0, a1)) as f32 * s;
7249        }
7250        acc
7251    }
7252}
7253
7254/// Two-activation q4 row via SDOT: the nibble unpack (the expensive
7255/// part) happens ONCE per group; both pre-quantized activations are
7256/// dotted against the same centered i8 registers. Per-lane math matches
7257/// `dot_q4_row_sdot` exactly.
7258#[cfg(target_arch = "aarch64")]
7259#[target_feature(enable = "neon,dotprod")]
7260unsafe fn dot_q4_row_sdot2(
7261    packed: &[u8],
7262    scales: &[u8],
7263    g0: usize,
7264    gpr: usize,
7265    xq1: &[i8],
7266    xq2: &[i8],
7267) -> (f32, f32) {
7268    // SAFETY: callers uphold slice-length contracts (16 packed bytes and
7269    // 2 scale bytes per group; xq*.len() == gpr·GROUP_SIZE).
7270    unsafe {
7271        use core::arch::aarch64::*;
7272        use core::arch::asm;
7273        let lomask = vdupq_n_u8(0x0F);
7274        let eight = vdupq_n_s8(8);
7275        let (mut acc1, mut acc2) = (0f32, 0f32);
7276        for gi in 0..gpr {
7277            let g = g0 + gi;
7278            let s = f16_to_f32(u16::from_le_bytes([scales[g * 2], scales[g * 2 + 1]]));
7279            let b = vld1q_u8(packed.as_ptr().add(g * 16));
7280            let lo = vandq_u8(b, lomask);
7281            let hi = vshrq_n_u8::<4>(b);
7282            let e0 = vsubq_s8(vreinterpretq_s8_u8(vzip1q_u8(lo, hi)), eight);
7283            let e1 = vsubq_s8(vreinterpretq_s8_u8(vzip2q_u8(lo, hi)), eight);
7284            let x10 = vld1q_s8(xq1.as_ptr().add(gi * GROUP_SIZE));
7285            let x11 = vld1q_s8(xq1.as_ptr().add(gi * GROUP_SIZE + 16));
7286            let x20 = vld1q_s8(xq2.as_ptr().add(gi * GROUP_SIZE));
7287            let x21 = vld1q_s8(xq2.as_ptr().add(gi * GROUP_SIZE + 16));
7288            let (mut a0, mut a1, mut b0, mut b1) = (
7289                vdupq_n_s32(0),
7290                vdupq_n_s32(0),
7291                vdupq_n_s32(0),
7292                vdupq_n_s32(0),
7293            );
7294            asm!(
7295                "sdot {a0:v}.4s, {e0:v}.16b, {x10:v}.16b",
7296                "sdot {a1:v}.4s, {e1:v}.16b, {x11:v}.16b",
7297                "sdot {b0:v}.4s, {e0:v}.16b, {x20:v}.16b",
7298                "sdot {b1:v}.4s, {e1:v}.16b, {x21:v}.16b",
7299                a0 = inout(vreg) a0, a1 = inout(vreg) a1,
7300                b0 = inout(vreg) b0, b1 = inout(vreg) b1,
7301                e0 = in(vreg) e0, e1 = in(vreg) e1,
7302                x10 = in(vreg) x10, x11 = in(vreg) x11,
7303                x20 = in(vreg) x20, x21 = in(vreg) x21,
7304                options(pure, nomem, nostack),
7305            );
7306            acc1 += vaddvq_s32(vaddq_s32(a0, a1)) as f32 * s;
7307            acc2 += vaddvq_s32(vaddq_s32(b0, b1)) as f32 * s;
7308        }
7309        (acc1, acc2)
7310    }
7311}
7312
7313// ───────────────────── fused int8 kernels ─────────────────────
7314
7315/// `acc += w · row` where the row is centered i8 — NEON widen+fma on
7316/// aarch64, scalar elsewhere. The KV-cache q8 value path rides on this.
7317#[inline]
7318pub(crate) fn axpy_i8_f32(acc: &mut [f32], row: &[i8], w: f32) {
7319    #[cfg(target_arch = "aarch64")]
7320    unsafe {
7321        return axpy_i8_f32_neon(acc, row, w);
7322    }
7323    #[cfg(target_arch = "x86_64")]
7324    if avx2_enabled() {
7325        return unsafe { axpy_i8_f32_avx2(acc, row, w) };
7326    }
7327    #[allow(unreachable_code)]
7328    {
7329        for (a, &b) in acc.iter_mut().zip(row) {
7330            *a += w * b as f32;
7331        }
7332    }
7333}
7334
7335/// i8→f32 axpy via AVX2/FMA (x86 mirror of `axpy_i8_f32_neon`).
7336#[cfg(target_arch = "x86_64")]
7337#[target_feature(enable = "avx2,fma")]
7338unsafe fn axpy_i8_f32_avx2(acc: &mut [f32], row: &[i8], w: f32) {
7339    // SAFETY: callers uphold slice-length contracts (see call sites).
7340    unsafe {
7341        use core::arch::x86_64::*;
7342        let n = acc.len().min(row.len());
7343        let ap = acc.as_mut_ptr();
7344        let rp = row.as_ptr();
7345        let wv = _mm256_set1_ps(w);
7346        let mut j = 0usize;
7347        while j + 16 <= n {
7348            let rb = _mm_loadu_si128(rp.add(j) as *const __m128i);
7349            let lo = _mm256_cvtepi32_ps(_mm256_cvtepi8_epi32(rb));
7350            let hi = _mm256_cvtepi32_ps(_mm256_cvtepi8_epi32(_mm_srli_si128::<8>(rb)));
7351            let v0 = _mm256_fmadd_ps(wv, lo, _mm256_loadu_ps(ap.add(j)));
7352            let v1 = _mm256_fmadd_ps(wv, hi, _mm256_loadu_ps(ap.add(j + 8)));
7353            _mm256_storeu_ps(ap.add(j), v0);
7354            _mm256_storeu_ps(ap.add(j + 8), v1);
7355            j += 16;
7356        }
7357        while j < n {
7358            *ap.add(j) += w * (*rp.add(j)) as f32;
7359            j += 1;
7360        }
7361    }
7362}
7363
7364#[cfg(target_arch = "aarch64")]
7365#[target_feature(enable = "neon")]
7366unsafe fn axpy_i8_f32_neon(acc: &mut [f32], row: &[i8], w: f32) {
7367    // SAFETY: callers uphold slice-length contracts (see call sites).
7368    unsafe {
7369        use core::arch::aarch64::*;
7370        let n = acc.len().min(row.len());
7371        let ap = acc.as_mut_ptr();
7372        let rp = row.as_ptr();
7373        let wv = vdupq_n_f32(w);
7374        let mut j = 0usize;
7375        while j + 16 <= n {
7376            let rb = vld1q_s8(rp.add(j));
7377            let lo = vmovl_s8(vget_low_s8(rb));
7378            let hi = vmovl_s8(vget_high_s8(rb));
7379            for (off, half) in [(0, lo), (8, hi)] {
7380                let f0 = vcvtq_f32_s32(vmovl_s16(vget_low_s16(half)));
7381                let f1 = vcvtq_f32_s32(vmovl_s16(vget_high_s16(half)));
7382                let o = j + off;
7383                vst1q_f32(ap.add(o), vfmaq_f32(vld1q_f32(ap.add(o)), wv, f0));
7384                vst1q_f32(ap.add(o + 4), vfmaq_f32(vld1q_f32(ap.add(o + 4)), wv, f1));
7385            }
7386            j += 16;
7387        }
7388        while j < n {
7389            *ap.add(j) += w * (*rp.add(j)) as f32;
7390            j += 1;
7391        }
7392    }
7393}
7394
7395/// i8 row · f32 x. NEON on aarch64 (ported from vmfcore `dot_i8_f32_neon`,
7396/// ≈9× scalar), scalar elsewhere.
7397#[inline]
7398pub(crate) fn dot_i8_f32(w: &[u8], x: &[f32]) -> f32 {
7399    #[cfg(target_arch = "aarch64")]
7400    unsafe {
7401        return dot_i8_f32_neon(w, x);
7402    }
7403    #[cfg(target_arch = "x86_64")]
7404    if avx2_enabled() {
7405        return unsafe { dot_i8_f32_avx2(w, x) };
7406    }
7407    #[allow(unreachable_code)]
7408    {
7409        let mut sum = 0.0f32;
7410        for (j, &b) in w.iter().enumerate() {
7411            sum += (b as i8) as f32 * x[j];
7412        }
7413        sum
7414    }
7415}
7416
7417/// i8 row · (x ⊙ col_field) — the q8_2f row dot with the θ col-field
7418/// folded into the product (no prescaled copy of x). NEON on aarch64,
7419/// scalar elsewhere. Used by the active-neuron path `row_dot`.
7420#[inline]
7421fn dot_i8_col_f32(w: &[u8], x: &[f32], col: &[f32]) -> f32 {
7422    #[cfg(target_arch = "aarch64")]
7423    unsafe {
7424        return dot_i8_col_f32_neon(w, x, col);
7425    }
7426    #[allow(unreachable_code)]
7427    {
7428        let mut sum = 0.0f32;
7429        for (j, &b) in w.iter().enumerate() {
7430            sum += (b as i8) as f32 * x[j] * col[j];
7431        }
7432        sum
7433    }
7434}
7435
7436#[cfg(target_arch = "aarch64")]
7437#[target_feature(enable = "neon")]
7438unsafe fn dot_i8_col_f32_neon(w: &[u8], x: &[f32], col: &[f32]) -> f32 {
7439    // SAFETY: callers uphold slice-length contracts (see call sites).
7440    unsafe {
7441        use core::arch::aarch64::*;
7442        let n = x.len();
7443        let wp = w.as_ptr() as *const i8;
7444        let xp = x.as_ptr();
7445        let cp = col.as_ptr();
7446        let (mut a0, mut a1, mut a2, mut a3) = (
7447            vdupq_n_f32(0.0),
7448            vdupq_n_f32(0.0),
7449            vdupq_n_f32(0.0),
7450            vdupq_n_f32(0.0),
7451        );
7452        let mut j = 0usize;
7453        while j + 16 <= n {
7454            let wb = vld1q_s8(wp.add(j));
7455            let lo = vmovl_s8(vget_low_s8(wb));
7456            let hi = vmovl_s8(vget_high_s8(wb));
7457            let w0 = vcvtq_f32_s32(vmovl_s16(vget_low_s16(lo)));
7458            let w1 = vcvtq_f32_s32(vmovl_s16(vget_high_s16(lo)));
7459            let w2 = vcvtq_f32_s32(vmovl_s16(vget_low_s16(hi)));
7460            let w3 = vcvtq_f32_s32(vmovl_s16(vget_high_s16(hi)));
7461            a0 = vfmaq_f32(
7462                a0,
7463                w0,
7464                vmulq_f32(vld1q_f32(xp.add(j)), vld1q_f32(cp.add(j))),
7465            );
7466            a1 = vfmaq_f32(
7467                a1,
7468                w1,
7469                vmulq_f32(vld1q_f32(xp.add(j + 4)), vld1q_f32(cp.add(j + 4))),
7470            );
7471            a2 = vfmaq_f32(
7472                a2,
7473                w2,
7474                vmulq_f32(vld1q_f32(xp.add(j + 8)), vld1q_f32(cp.add(j + 8))),
7475            );
7476            a3 = vfmaq_f32(
7477                a3,
7478                w3,
7479                vmulq_f32(vld1q_f32(xp.add(j + 12)), vld1q_f32(cp.add(j + 12))),
7480            );
7481            j += 16;
7482        }
7483        let mut sum = vaddvq_f32(vaddq_f32(vaddq_f32(a0, a1), vaddq_f32(a2, a3)));
7484        while j < n {
7485            sum += (*wp.add(j)) as f32 * *xp.add(j) * *cp.add(j);
7486            j += 1;
7487        }
7488        sum
7489    }
7490}
7491
7492#[cfg(target_arch = "aarch64")]
7493#[target_feature(enable = "neon")]
7494unsafe fn dot_i8_f32_neon(w: &[u8], x: &[f32]) -> f32 {
7495    // SAFETY: callers uphold slice-length contracts (see call sites).
7496    unsafe {
7497        use core::arch::aarch64::*;
7498        let n = x.len();
7499        let wp = w.as_ptr() as *const i8;
7500        let xp = x.as_ptr();
7501        let (mut a0, mut a1, mut a2, mut a3) = (
7502            vdupq_n_f32(0.0),
7503            vdupq_n_f32(0.0),
7504            vdupq_n_f32(0.0),
7505            vdupq_n_f32(0.0),
7506        );
7507        let mut j = 0usize;
7508        while j + 16 <= n {
7509            let wb = vld1q_s8(wp.add(j));
7510            let lo = vmovl_s8(vget_low_s8(wb));
7511            let hi = vmovl_s8(vget_high_s8(wb));
7512            let w0 = vcvtq_f32_s32(vmovl_s16(vget_low_s16(lo)));
7513            let w1 = vcvtq_f32_s32(vmovl_s16(vget_high_s16(lo)));
7514            let w2 = vcvtq_f32_s32(vmovl_s16(vget_low_s16(hi)));
7515            let w3 = vcvtq_f32_s32(vmovl_s16(vget_high_s16(hi)));
7516            a0 = vfmaq_f32(a0, w0, vld1q_f32(xp.add(j)));
7517            a1 = vfmaq_f32(a1, w1, vld1q_f32(xp.add(j + 4)));
7518            a2 = vfmaq_f32(a2, w2, vld1q_f32(xp.add(j + 8)));
7519            a3 = vfmaq_f32(a3, w3, vld1q_f32(xp.add(j + 12)));
7520            j += 16;
7521        }
7522        let mut sum = vaddvq_f32(vaddq_f32(vaddq_f32(a0, a1), vaddq_f32(a2, a3)));
7523        while j < n {
7524            sum += (*wp.add(j)) as f32 * *xp.add(j);
7525            j += 1;
7526        }
7527        sum
7528    }
7529}
7530
7531#[allow(clippy::too_many_arguments)]
7532fn qmatvec(
7533    q: &[u8],
7534    rep: &[u8],
7535    row_scale: &[f32],
7536    x: &[f32],
7537    col_field: &[f32],
7538    dtype: TensorDtype,
7539    rows: usize,
7540    cols: usize,
7541    out: &mut [f32],
7542    pool: Option<&Pool>,
7543) {
7544    debug_assert_eq!(out.len(), rows);
7545    #[cfg(not(target_arch = "aarch64"))]
7546    let _ = rep;
7547
7548    #[cfg(target_arch = "aarch64")]
7549    if sdot_enabled() {
7550        let act = if dtype == TensorDtype::Q8_2f {
7551            split_act_q8_2f(x, col_field)
7552        } else {
7553            split_act(x)
7554        };
7555        let out_addr = SendMut(out.as_mut_ptr());
7556        let run_range = |start: usize, end: usize| {
7557            q8_range_sdot(q, rep, row_scale, &act, cols, out_addr, start, end)
7558        };
7559        match pool {
7560            Some(pool) if rows >= 256 => pool.run_rows(rows, &run_range),
7561            _ => run_range(0, rows),
7562        }
7563        return;
7564    }
7565    // x86 A8W8 via AVX2 maddubs — same quantized-activation contract as
7566    // the SDOT path (CMF_AVX2=0 keeps the exact i8×f32 loop).
7567    #[cfg(target_arch = "x86_64")]
7568    if avx2_a8w8_enabled() {
7569        let act = if dtype == TensorDtype::Q8_2f {
7570            split_act_q8_2f(x, col_field)
7571        } else {
7572            split_act(x)
7573        };
7574        let out_addr = SendMut(out.as_mut_ptr());
7575        let run_range = |start: usize, end: usize| {
7576            q8_range_avx2(q, row_scale, &act, cols, out_addr, start, end)
7577        };
7578        match pool {
7579            Some(pool) if rows >= 256 => pool.run_rows(rows, &run_range),
7580            _ => run_range(0, rows),
7581        }
7582        return;
7583    }
7584
7585    prescale_with(x, col_field, dtype, 1, |xs| {
7586        let out_addr = SendMut(out.as_mut_ptr());
7587        let run_range = move |start: usize, end: usize| {
7588            for o in start..end {
7589                let v = dot_i8_f32(&q[o * cols..(o + 1) * cols], xs) * row_scale[o];
7590                // SAFETY: disjoint row ranges per worker.
7591                unsafe { *out_addr.at(o) = v };
7592            }
7593        };
7594        match pool {
7595            Some(pool) if rows >= 256 => pool.run_rows(rows, &run_range),
7596            _ => run_range(0, rows),
7597        }
7598    });
7599}
7600
7601#[allow(clippy::too_many_arguments)]
7602fn qmatvec2(
7603    q: &[u8],
7604    row_scale: &[f32],
7605    x1: &[f32],
7606    x2: &[f32],
7607    col_field: &[f32],
7608    dtype: TensorDtype,
7609    rows: usize,
7610    cols: usize,
7611    o1: &mut [f32],
7612    o2: &mut [f32],
7613    pool: Option<&Pool>,
7614) {
7615    #[cfg(target_arch = "aarch64")]
7616    if sdot_enabled() {
7617        let a1s = if dtype == TensorDtype::Q8_2f {
7618            split_act_q8_2f(x1, col_field)
7619        } else {
7620            split_act(x1)
7621        };
7622        let a2s = if dtype == TensorDtype::Q8_2f {
7623            split_act_q8_2f(x2, col_field)
7624        } else {
7625            split_act(x2)
7626        };
7627        let p1 = SendMut(o1.as_mut_ptr());
7628        let p2 = SendMut(o2.as_mut_ptr());
7629        let run_range = |start: usize, end: usize| {
7630            q8_range2_sdot(q, row_scale, &a1s, &a2s, cols, p1, p2, start, end)
7631        };
7632        match pool {
7633            Some(pool) if rows >= 256 => pool.run_rows(rows, &run_range),
7634            _ => run_range(0, rows),
7635        }
7636        return;
7637    }
7638    #[cfg(target_arch = "x86_64")]
7639    if avx2_a8w8_enabled() {
7640        let a1s = if dtype == TensorDtype::Q8_2f {
7641            split_act_q8_2f(x1, col_field)
7642        } else {
7643            split_act(x1)
7644        };
7645        let a2s = if dtype == TensorDtype::Q8_2f {
7646            split_act_q8_2f(x2, col_field)
7647        } else {
7648            split_act(x2)
7649        };
7650        let p1 = SendMut(o1.as_mut_ptr());
7651        let p2 = SendMut(o2.as_mut_ptr());
7652        let run_range = |start: usize, end: usize| {
7653            q8_range2_avx2(q, row_scale, &a1s, &a2s, cols, p1, p2, start, end)
7654        };
7655        match pool {
7656            Some(pool) if rows >= 256 => pool.run_rows(rows, &run_range),
7657            _ => run_range(0, rows),
7658        }
7659        return;
7660    }
7661
7662    prescale_with(x1, col_field, dtype, 1, |x1s| {
7663        prescale_with(x2, col_field, dtype, 2, |x2s| {
7664            let p1 = SendMut(o1.as_mut_ptr());
7665            let p2 = SendMut(o2.as_mut_ptr());
7666            let run_range = move |start: usize, end: usize| {
7667                for o in start..end {
7668                    let row = &q[o * cols..(o + 1) * cols];
7669                    let s1 = dot_i8_f32(row, x1s) * row_scale[o];
7670                    let s2 = dot_i8_f32(row, x2s) * row_scale[o];
7671                    // SAFETY: disjoint row ranges per worker.
7672                    unsafe {
7673                        *p1.at(o) = s1;
7674                        *p2.at(o) = s2;
7675                    }
7676                }
7677            };
7678            match pool {
7679                Some(pool) if rows >= 256 => pool.run_rows(rows, &run_range),
7680                _ => run_range(0, rows),
7681            }
7682        });
7683    });
7684}
7685
7686#[derive(Clone, Copy)]
7687struct SendMut(*mut f32);
7688unsafe impl Send for SendMut {}
7689unsafe impl Sync for SendMut {}
7690
7691impl SendMut {
7692    #[inline]
7693    fn at(self, i: usize) -> *mut f32 {
7694        unsafe { self.0.add(i) }
7695    }
7696}
7697
7698#[cfg(test)]
7699mod tests {
7700    use super::*;
7701
7702    #[test]
7703    fn f32_matvec_matches_matvec_rows_bitexact() {
7704        let (rows, cols) = (300, 40);
7705        let w: Vec<f32> = (0..rows * cols).map(|i| (i as f32 * 0.017).sin()).collect();
7706        let x: Vec<f32> = (0..cols).map(|i| (i as f32 * 0.05).cos()).collect();
7707        let qt = QTensor::from_f32(w.clone(), rows, cols);
7708
7709        let mut a = vec![0.0f32; rows];
7710        matvec_rows(None, &w, &x, &mut a);
7711        let mut b = vec![0.0f32; rows];
7712        qt.matvec(&x, &mut b, None);
7713        assert_eq!(a, b);
7714    }
7715
7716    #[test]
7717    fn sdot_kernel_exact_on_grid() {
7718        // Activations already on the i8 grid (±1 with amax=1 → sx=1/127,
7719        // xq=±127 dequantizes EXACTLY) → the SDOT path must match the
7720        // exact f32 dot to float rounding. This isolates kernel
7721        // correctness from quantization noise.
7722        eprintln!("sdot_enabled = {}", sdot_enabled());
7723        let (rows, cols) = (9, 80); // odd rows → exercises 4-row + tail
7724        let w: Vec<u8> = (0..rows * cols)
7725            .map(|i| (((i * 37) % 251) as i32 - 125) as i8 as u8)
7726            .collect();
7727        let scales: Vec<f32> = (0..rows).map(|o| 0.005 + o as f32 * 0.001).collect();
7728        let x: Vec<f32> = (0..cols)
7729            .map(|i| match i % 3 {
7730                0 => 1.0,
7731                1 => -1.0,
7732                _ => 0.0,
7733            })
7734            .collect();
7735        let mut a = vec![0.0f32; rows];
7736        qmatvec(
7737            &w,
7738            &[],
7739            &scales,
7740            &x,
7741            &[],
7742            TensorDtype::Q8Row,
7743            rows,
7744            cols,
7745            &mut a,
7746            None,
7747        );
7748        for o in 0..rows {
7749            let mut acc = 0.0f32;
7750            for j in 0..cols {
7751                acc += (w[o * cols + j] as i8) as f32 * x[j];
7752            }
7753            let expect = acc * scales[o];
7754            assert!(
7755                (a[o] - expect).abs() < 1e-3 * expect.abs().max(1e-3),
7756                "row {o}: {} vs {expect}",
7757                a[o]
7758            );
7759        }
7760    }
7761
7762    #[test]
7763    fn q1_tbl_fast_path_matches_reference() {
7764        // gpr = 8 exercises the TBL pair-load fast loop, and the LAST
7765        // row's final 4-tile window trips the 4B-overread guard (the
7766        // payload ends exactly at the last tile) — both paths must
7767        // agree with the dequant reference.
7768        let (rows, cols) = (5, 256);
7769        let gpr = cols / GROUP_SIZE;
7770        let mut bytes = Vec::new();
7771        for t in 0..rows * gpr {
7772            let s = 0.007 + (t % 11) as f32 * 0.004;
7773            bytes.extend_from_slice(&cortiq_core::quant::f32_to_f16(s).to_le_bytes());
7774            for j in 0..4 {
7775                bytes.push(((t * 53 + j * 89 + 7) % 249) as u8);
7776            }
7777        }
7778        let x: Vec<f32> = (0..cols)
7779            .map(|i| if (i * 5) % 7 < 3 { 1.0 } else { -1.0 })
7780            .collect();
7781        let mut w = vec![0.0f32; rows * cols];
7782        cortiq_core::quant::dequant_q1(&bytes, &mut w);
7783        let mut got = vec![0.0f32; rows];
7784        q1_matvec(&bytes, &x, rows, cols, &mut got, None);
7785        for o in 0..rows {
7786            let expect: f32 = (0..cols).map(|j| w[o * cols + j] * x[j]).sum();
7787            assert!(
7788                (got[o] - expect).abs() < 1e-3 * expect.abs().max(1e-3),
7789                "row {o}: {} vs {expect}",
7790                got[o]
7791            );
7792        }
7793        // Blocked 1×4 batch (b=5: one quad + remainder) must equal the
7794        // single-matvec path bit-for-bit.
7795        let b = 5usize;
7796        let mut xs_all = Vec::new();
7797        for bi in 0..b {
7798            xs_all.extend(x.iter().map(|v| if bi % 2 == 0 { *v } else { -*v }));
7799        }
7800        let mut mm = vec![0.0f32; b * rows];
7801        q1_matmat(&bytes, &xs_all, b, rows, cols, &mut mm, None);
7802        for bi in 0..b {
7803            let mut single = vec![0.0f32; rows];
7804            q1_matvec(
7805                &bytes,
7806                &xs_all[bi * cols..(bi + 1) * cols],
7807                rows,
7808                cols,
7809                &mut single,
7810                None,
7811            );
7812            assert_eq!(&mm[bi * rows..(bi + 1) * rows], &single[..], "stream {bi}");
7813        }
7814    }
7815
7816    #[test]
7817    fn q1_kernels_match_exact_reference() {
7818        // Synthetic q1 payload: 6-byte tiles [f16 scale][4B bits].
7819        let (rows, cols) = (7, 96);
7820        let gpr = cols / GROUP_SIZE;
7821        let mut bytes = Vec::new();
7822        for t in 0..rows * gpr {
7823            let s = 0.01 + (t % 13) as f32 * 0.003;
7824            bytes.extend_from_slice(&cortiq_core::quant::f32_to_f16(s).to_le_bytes());
7825            for j in 0..4 {
7826                bytes.push(((t * 31 + j * 97) % 251) as u8);
7827            }
7828        }
7829        // On-grid activations (±1, amax 1) → the SDOT path is exact.
7830        let x: Vec<f32> = (0..cols)
7831            .map(|i| if i % 3 == 0 { 1.0 } else { -1.0 })
7832            .collect();
7833        // Reference through the core dequant.
7834        let mut w = vec![0.0f32; rows * cols];
7835        cortiq_core::quant::dequant_q1(&bytes, &mut w);
7836        let mut expect = vec![0.0f32; rows];
7837        for o in 0..rows {
7838            expect[o] = (0..cols).map(|j| w[o * cols + j] * x[j]).sum();
7839        }
7840        let mut got = vec![0.0f32; rows];
7841        q1_matvec(&bytes, &x, rows, cols, &mut got, None);
7842        for o in 0..rows {
7843            assert!(
7844                (got[o] - expect[o]).abs() < 1e-3 * expect[o].abs().max(1e-3),
7845                "row {o}: {} vs {}",
7846                got[o],
7847                expect[o]
7848            );
7849        }
7850        // Pair and batch paths agree with the single path.
7851        let x2: Vec<f32> = x.iter().map(|v| -v).collect();
7852        let (mut a1, mut a2) = (vec![0.0f32; rows], vec![0.0f32; rows]);
7853        q1_matvec2(&bytes, &x, &x2, rows, cols, &mut a1, &mut a2, None);
7854        assert_eq!(a1, got);
7855        let mut xs = x.clone();
7856        xs.extend_from_slice(&x2);
7857        let mut mm = vec![0.0f32; 2 * rows];
7858        q1_matmat(&bytes, &xs, 2, rows, cols, &mut mm, None);
7859        assert_eq!(&mm[..rows], got.as_slice());
7860        assert_eq!(&mm[rows..], a2.as_slice());
7861    }
7862
7863    #[test]
7864    fn repack_is_bit_identical() {
7865        // The interleaved-repack kernel must produce EXACTLY the same
7866        // bits as the mmap-layout kernel: integer accumulation is order-
7867        // exact, the f32 epilogue is identical. Odd rows exercise the
7868        // tail; direct range calls exercise unaligned pool splits.
7869        let (rows, cols) = (267, 96); // 66 groups + 3 tail rows, cols % 16 == 0
7870        let w: Vec<u8> = (0..rows * cols)
7871            .map(|i| (((i * 89) % 253) as i32 - 126) as i8 as u8)
7872            .collect();
7873        let scales: Vec<f32> = (0..rows).map(|o| 0.003 + o as f32 * 0.0007).collect();
7874        let x: Vec<f32> = (0..cols).map(|i| (i as f32 * 0.37).sin() * 2.0).collect();
7875        let rep = q8_repack_layout(&w, rows, cols);
7876        // Group interleave round-trips.
7877        for g in 0..rows / 4 {
7878            for c in 0..cols / 16 {
7879                for lane in 0..4 {
7880                    assert_eq!(
7881                        &rep[g * 4 * cols + c * 64 + lane * 16
7882                            ..g * 4 * cols + c * 64 + lane * 16 + 16],
7883                        &w[(g * 4 + lane) * cols + c * 16..(g * 4 + lane) * cols + c * 16 + 16],
7884                    );
7885                }
7886            }
7887        }
7888        let mut a = vec![0.0f32; rows];
7889        qmatvec(
7890            &w,
7891            &[],
7892            &scales,
7893            &x,
7894            &[],
7895            TensorDtype::Q8Row,
7896            rows,
7897            cols,
7898            &mut a,
7899            None,
7900        );
7901        let mut b = vec![0.0f32; rows];
7902        qmatvec(
7903            &w,
7904            &rep,
7905            &scales,
7906            &x,
7907            &[],
7908            TensorDtype::Q8Row,
7909            rows,
7910            cols,
7911            &mut b,
7912            None,
7913        );
7914        assert_eq!(a, b, "full-range repack output diverged");
7915
7916        #[cfg(target_arch = "aarch64")]
7917        if sdot_enabled() {
7918            // Unaligned range split (pool workers get arbitrary bounds).
7919            let act = split_act(&x);
7920            let mut c1 = vec![0.0f32; rows];
7921            let mut c2 = vec![0.0f32; rows];
7922            q8_range_sdot(
7923                &w,
7924                &[],
7925                &scales,
7926                &act,
7927                cols,
7928                SendMut(c1.as_mut_ptr()),
7929                3,
7930                rows - 2,
7931            );
7932            q8_range_sdot(
7933                &w,
7934                &rep,
7935                &scales,
7936                &act,
7937                cols,
7938                SendMut(c2.as_mut_ptr()),
7939                3,
7940                rows - 2,
7941            );
7942            assert_eq!(c1, c2, "unaligned-range repack output diverged");
7943        }
7944    }
7945
7946    #[test]
7947    fn sdot_a8w8_noise_is_bounded() {
7948        // Off-grid activations: A8 quantization noise must stay small in
7949        // relative L2 over the whole output (realistic accuracy contract;
7950        // vmfcore measured argmax-identical decode on real models).
7951        let (rows, cols) = (16, 512);
7952        let w: Vec<u8> = (0..rows * cols)
7953            .map(|i| (((i * 37) % 251) as i32 - 125) as i8 as u8)
7954            .collect();
7955        let scales = vec![0.01f32; rows];
7956        let x: Vec<f32> = (0..cols).map(|i| (i as f32 * 0.21).sin()).collect();
7957        let mut a = vec![0.0f32; rows];
7958        qmatvec(
7959            &w,
7960            &[],
7961            &scales,
7962            &x,
7963            &[],
7964            TensorDtype::Q8Row,
7965            rows,
7966            cols,
7967            &mut a,
7968            None,
7969        );
7970        let (mut num, mut den) = (0f64, 0f64);
7971        for o in 0..rows {
7972            let mut acc = 0.0f32;
7973            for j in 0..cols {
7974                acc += (w[o * cols + j] as i8) as f32 * x[j];
7975            }
7976            let expect = acc * scales[o];
7977            num += ((a[o] - expect) as f64).powi(2);
7978            den += (expect as f64).powi(2);
7979        }
7980        let rel = (num / den.max(1e-12)).sqrt();
7981        assert!(rel < 0.05, "A8W8 relative L2 error too high: {rel}");
7982    }
7983
7984    #[test]
7985    fn i8_dot_neon_matches_scalar() {
7986        let n = 100;
7987        let w: Vec<u8> = (0..n).map(|i| ((i * 37 + 11) % 251) as u8).collect();
7988        let x: Vec<f32> = (0..n).map(|i| (i as f32 * 0.13).sin()).collect();
7989        let mut scalar = 0.0f32;
7990        for j in 0..n {
7991            scalar += (w[j] as i8) as f32 * x[j];
7992        }
7993        let fast = dot_i8_f32(&w, &x);
7994        assert!((scalar - fast).abs() < 1e-3 * scalar.abs().max(1.0));
7995    }
7996
7997    /// Fused vbit matvec must match full dequant_vbit + dense matvec.
7998    #[test]
7999    fn vbitmatvec_matches_full_dequant() {
8000        let (rows, cols) = (6, 64);
8001        let ng = cols / GROUP_SIZE;
8002        // Hand-craft: bits per row, f16 scales, packed rows.
8003        let bits: Vec<u8> = vec![3, 4, 5, 6, 8, 4];
8004        let mut bytes = bits.clone();
8005        for g in 0..rows * ng {
8006            let s = 0.02 + 0.001 * g as f32;
8007            bytes.extend_from_slice(&cortiq_core::quant::f32_to_f16(s).to_le_bytes());
8008        }
8009        for r in 0..rows {
8010            let b = bits[r] as usize;
8011            let (mut acc, mut nb) = (0u64, 0usize);
8012            let mut rowbytes = Vec::new();
8013            for i in 0..cols {
8014                let v = ((i * 7 + r * 13) % (1 << b)) as u64;
8015                acc = (acc << b) | v;
8016                nb += b;
8017                while nb >= 8 {
8018                    nb -= 8;
8019                    rowbytes.push(((acc >> nb) & 0xFF) as u8);
8020                }
8021            }
8022            if nb > 0 {
8023                rowbytes.push(((acc << (8 - nb)) & 0xFF) as u8);
8024            }
8025            bytes.extend_from_slice(&rowbytes);
8026        }
8027        let x: Vec<f32> = (0..cols).map(|i| (i as f32 * 0.19).sin()).collect();
8028
8029        let mut reference = vec![0f32; rows * cols];
8030        cortiq_core::quant::dequant_vbit(&bytes, rows, cols, &mut reference).unwrap();
8031        let mut expect = vec![0f32; rows];
8032        for r in 0..rows {
8033            expect[r] = reference[r * cols..(r + 1) * cols]
8034                .iter()
8035                .zip(&x)
8036                .map(|(w, xv)| w * xv)
8037                .sum();
8038        }
8039        let mut got = vec![0f32; rows];
8040        let offsets = vbit_row_offsets(&bytes, rows, cols);
8041        vbitmatvec(&bytes, &offsets, &x, rows, cols, &mut got, None);
8042        // SDOT path quantizes activations to i8 (A8W8): bounded noise,
8043        // same contract as q8 (exact path is pinned by CMF_SDOT=0 in
8044        // the golden-parity gate).
8045        let tol = if a8w8_enabled() { 6e-2 } else { 1e-4 };
8046        let scale = expect.iter().fold(0f32, |m, v| m.max(v.abs())).max(1e-6);
8047        for r in 0..rows {
8048            assert!(
8049                (got[r] - expect[r]).abs() < tol * scale,
8050                "row {r}: {} vs {}",
8051                got[r],
8052                expect[r]
8053            );
8054        }
8055    }
8056
8057    /// Fused q4 matvec must match the reference full-dequant + dense
8058    /// matvec bit-for-bit in structure (same f32 math, group order).
8059    /// vbit matmat: the blocked 1×4 leg must match the per-row path
8060    /// (paired env toggle; larger shape so both code paths engage).
8061    #[test]
8062    #[cfg(target_arch = "x86_64")]
8063    fn vbit_matmat_blocked_matches_per_row() {
8064        let (rows, cols, b) = (64usize, 128usize, 9usize);
8065        let ng = cols / GROUP_SIZE;
8066        let bits: Vec<u8> = (0..rows).map(|r| [3u8, 4, 5, 6][r % 4]).collect();
8067        let mut bytes = bits.clone();
8068        for g in 0..rows * ng {
8069            let sc = 0.02 + 0.0005 * g as f32;
8070            bytes.extend_from_slice(&cortiq_core::quant::f32_to_f16(sc).to_le_bytes());
8071        }
8072        for r in 0..rows {
8073            let bw = bits[r] as usize;
8074            let (mut acc, mut nb) = (0u64, 0usize);
8075            let mut rowbytes = Vec::new();
8076            for i in 0..cols {
8077                let v = ((i * 7 + r * 13) % (1 << bw)) as u64;
8078                acc = (acc << bw) | v;
8079                nb += bw;
8080                while nb >= 8 {
8081                    nb -= 8;
8082                    rowbytes.push(((acc >> nb) & 0xFF) as u8);
8083                }
8084            }
8085            if nb > 0 {
8086                rowbytes.push(((acc << (8 - nb)) & 0xFF) as u8);
8087            }
8088            bytes.extend_from_slice(&rowbytes);
8089        }
8090        let x: Vec<f32> = (0..b * cols)
8091            .map(|i| ((i * 13 + 7) % 97) as f32 / 97.0 - 0.5)
8092            .collect();
8093        let offsets = vbit_row_offsets(&bytes, rows, cols);
8094        let mut y_a = vec![0f32; b * rows];
8095        let mut y_b = vec![0f32; b * rows];
8096        unsafe { std::env::set_var("CMF_X86_BLOCKED", "1") };
8097        vbitmatmat(&bytes, &offsets, &x, b, rows, cols, &mut y_a, None);
8098        unsafe { std::env::set_var("CMF_X86_BLOCKED", "0") };
8099        vbitmatmat(&bytes, &offsets, &x, b, rows, cols, &mut y_b, None);
8100        unsafe { std::env::remove_var("CMF_X86_BLOCKED") };
8101        let max_d = y_a
8102            .iter()
8103            .zip(&y_b)
8104            .map(|(p, q)| (p - q).abs())
8105            .fold(0.0f32, f32::max);
8106        assert!(max_d < 1e-4, "vbit blocked ≠ per-row: max|Δ| = {max_d}");
8107    }
8108
8109    /// q4t blocked 1×4 (SDOT on ARM, AVX2 on x86) must equal the
8110    /// per-row path exactly: same nibble unpack, same group order,
8111    /// same f32 accumulation — batch == matvec bit-for-bit. b=9 covers
8112    /// two full 1×4 blocks plus a remainder through the single-row
8113    /// kernel. (Both paths produce identical output, so the shared
8114    /// CMF_X86_BLOCKED env var racing with other tests cannot flip
8115    /// the verdict — worst case both sides take the same path.)
8116    #[test]
8117    fn q4t_matmat_blocked_matches_per_row() {
8118        let (rows, cols, b) = (16usize, 64usize, 9usize);
8119        let gpr = cols / GROUP_SIZE;
8120        let mut bytes = vec![0u8; rows * gpr * Q4_TILE];
8121        for r in 0..rows {
8122            for g in 0..gpr {
8123                let t = (r * gpr + g) * Q4_TILE;
8124                let sc = 0.02 + 0.001 * (r * gpr + g) as f32;
8125                bytes[t..t + 2].copy_from_slice(&cortiq_core::quant::f32_to_f16(sc).to_le_bytes());
8126                for k in 0..16 {
8127                    bytes[t + 2 + k] = ((r * 31 + g * 7 + k * 13) % 251) as u8;
8128                }
8129            }
8130        }
8131        let x: Vec<f32> = (0..b * cols)
8132            .map(|i| ((i * 13 + 7) % 97) as f32 / 97.0 - 0.5)
8133            .collect();
8134        let mut y_blk = vec![0f32; b * rows];
8135        let mut y_row = vec![0f32; b * rows];
8136        unsafe { std::env::set_var("CMF_X86_BLOCKED", "1") };
8137        q4t_matmat(&bytes, &x, b, rows, cols, &mut y_blk, None);
8138        unsafe { std::env::set_var("CMF_X86_BLOCKED", "0") };
8139        q4t_matmat(&bytes, &x, b, rows, cols, &mut y_row, None);
8140        unsafe { std::env::remove_var("CMF_X86_BLOCKED") };
8141        assert_eq!(y_blk, y_row, "q4t blocked 1x4 ≠ per-row");
8142    }
8143
8144    /// The wide-batch Accelerate arm of q4t_matmat vs a brute-force
8145    /// f32 dequant matmul: both are f32 GEMMs, so only reduction
8146    /// order differs — tight tolerance.
8147    #[cfg(target_os = "macos")]
8148    #[test]
8149    fn q4t_matmat_accel_matches_dequant_reference() {
8150        if !accel_gemm_enabled() {
8151            return; // CMF_ACCEL=0
8152        }
8153        let (rows, cols, b) = (512usize, 1024usize, 8usize); // ≥500K → accel arm
8154        let gpr = cols / GROUP_SIZE;
8155        let mut bytes = vec![0u8; rows * gpr * Q4_TILE];
8156        for r in 0..rows {
8157            for g in 0..gpr {
8158                let t = (r * gpr + g) * Q4_TILE;
8159                let sc = 0.02 + 0.0005 * ((r * gpr + g) % 64) as f32;
8160                bytes[t..t + 2].copy_from_slice(&cortiq_core::quant::f32_to_f16(sc).to_le_bytes());
8161                for k in 0..16 {
8162                    bytes[t + 2 + k] = ((r * 31 + g * 7 + k * 13) % 251) as u8;
8163                }
8164            }
8165        }
8166        let x: Vec<f32> = (0..b * cols)
8167            .map(|i| ((i * 13 + 7) % 97) as f32 / 97.0 - 0.5)
8168            .collect();
8169        let mut got = vec![0f32; b * rows];
8170        q4t_matmat(&bytes, &x, b, rows, cols, &mut got, None);
8171        // Brute-force reference off the same tiles.
8172        let mut w = vec![0f32; rows * cols];
8173        for r in 0..rows {
8174            for g in 0..gpr {
8175                let t = (r * gpr + g) * Q4_TILE;
8176                let s = f16_to_f32(u16::from_le_bytes([bytes[t], bytes[t + 1]]));
8177                for (k, &bb) in bytes[t + 2..t + Q4_TILE].iter().enumerate() {
8178                    w[r * cols + g * GROUP_SIZE + k * 2] = ((bb & 0x0F) as f32 - 8.0) * s;
8179                    w[r * cols + g * GROUP_SIZE + k * 2 + 1] =
8180                        (((bb >> 4) & 0x0F) as f32 - 8.0) * s;
8181                }
8182            }
8183        }
8184        for bi in 0..b {
8185            for r in 0..rows {
8186                let want: f32 = (0..cols).map(|j| x[bi * cols + j] * w[r * cols + j]).sum();
8187                let d = (got[bi * rows + r] - want).abs();
8188                assert!(
8189                    d <= want.abs().max(1.0) * 1e-4,
8190                    "accel q4t GEMM diverged at ({bi},{r}): {} vs {want}",
8191                    got[bi * rows + r]
8192                );
8193            }
8194        }
8195    }
8196
8197    #[test]
8198    fn q4matvec_matches_full_dequant() {
8199        let (rows, cols) = (8, 64);
8200        let groups = rows * cols / GROUP_SIZE;
8201        // Hand-craft a q4_block blob: nibbles then f16 scales.
8202        let mut bytes = Vec::with_capacity(groups * 16 + groups * 2);
8203        for i in 0..groups * 16 {
8204            bytes.push((((i * 7 + 3) % 256) & 0xFF) as u8);
8205        }
8206        for g in 0..groups {
8207            let s = 0.01 + 0.003 * g as f32;
8208            bytes.extend_from_slice(&cortiq_core::quant::f32_to_f16(s).to_le_bytes());
8209        }
8210        let x: Vec<f32> = (0..cols).map(|i| (i as f32 * 0.17).sin()).collect();
8211
8212        let mut reference = vec![0.0f32; rows * cols];
8213        cortiq_core::quant::dequant_q4_block(&bytes, &mut reference);
8214        let mut expect = vec![0.0f32; rows];
8215        for r in 0..rows {
8216            expect[r] = reference[r * cols..(r + 1) * cols]
8217                .iter()
8218                .zip(&x)
8219                .map(|(w, xv)| w * xv)
8220                .sum();
8221        }
8222
8223        let mut got = vec![0.0f32; rows];
8224        q4matvec(&bytes, &x, rows, cols, &mut got, None);
8225        // SDOT path quantizes activations to i8 (A8W8): bounded noise,
8226        // same contract as q8/vbit (exact path is pinned by CMF_SDOT=0
8227        // in the golden-parity gate).
8228        let tol = if a8w8_enabled() { 6e-2 } else { 1e-4 };
8229        let scale = expect.iter().fold(0f32, |m, v| m.max(v.abs())).max(1.0);
8230        for r in 0..rows {
8231            assert!(
8232                (got[r] - expect[r]).abs() < tol * scale,
8233                "row {r}: {} vs {}",
8234                got[r],
8235                expect[r]
8236            );
8237        }
8238    }
8239
8240    /// Fused two-input vbit matvec must equal two single matvecs exactly
8241    /// (same per-lane accumulation order on both scalar and SDOT paths).
8242    #[test]
8243    fn vbitmatvec2_equals_two_singles() {
8244        let (rows, cols) = (6, 64);
8245        let ng = cols / GROUP_SIZE;
8246        let bits: Vec<u8> = vec![3, 4, 5, 6, 8, 4];
8247        let mut bytes = bits.clone();
8248        for g in 0..rows * ng {
8249            let s = 0.02 + 0.001 * g as f32;
8250            bytes.extend_from_slice(&cortiq_core::quant::f32_to_f16(s).to_le_bytes());
8251        }
8252        for r in 0..rows {
8253            let b = bits[r] as usize;
8254            let (mut acc, mut nb) = (0u64, 0usize);
8255            let mut rowbytes = Vec::new();
8256            for i in 0..cols {
8257                let v = ((i * 7 + r * 13) % (1 << b)) as u64;
8258                acc = (acc << b) | v;
8259                nb += b;
8260                while nb >= 8 {
8261                    nb -= 8;
8262                    rowbytes.push(((acc >> nb) & 0xFF) as u8);
8263                }
8264            }
8265            if nb > 0 {
8266                rowbytes.push(((acc << (8 - nb)) & 0xFF) as u8);
8267            }
8268            bytes.extend_from_slice(&rowbytes);
8269        }
8270        let x1: Vec<f32> = (0..cols).map(|i| (i as f32 * 0.19).sin()).collect();
8271        let x2: Vec<f32> = (0..cols).map(|i| (i as f32 * 0.11).cos()).collect();
8272        let offsets = vbit_row_offsets(&bytes, rows, cols);
8273
8274        let (mut a1, mut a2) = (vec![0f32; rows], vec![0f32; rows]);
8275        vbitmatvec(&bytes, &offsets, &x1, rows, cols, &mut a1, None);
8276        vbitmatvec(&bytes, &offsets, &x2, rows, cols, &mut a2, None);
8277        let (mut b1, mut b2) = (vec![0f32; rows], vec![0f32; rows]);
8278        vbitmatvec2(
8279            &bytes, &offsets, &x1, &x2, rows, cols, &mut b1, &mut b2, None,
8280        );
8281        assert_eq!(a1, b1, "fused vbit lane 1 must be bit-identical");
8282        assert_eq!(a2, b2, "fused vbit lane 2 must be bit-identical");
8283    }
8284
8285    /// Fused two-input q4 matvec must equal two single matvecs exactly.
8286    #[test]
8287    fn q4matvec2_equals_two_singles() {
8288        let (rows, cols) = (8, 128);
8289        let groups = rows * cols / GROUP_SIZE;
8290        let mut bytes = Vec::with_capacity(groups * 16 + groups * 2);
8291        for i in 0..groups * 16 {
8292            bytes.push((((i * 7 + 3) % 256) & 0xFF) as u8);
8293        }
8294        for g in 0..groups {
8295            let s = 0.01 + 0.003 * g as f32;
8296            bytes.extend_from_slice(&cortiq_core::quant::f32_to_f16(s).to_le_bytes());
8297        }
8298        // Include an outlier channel so the SDOT correction path is
8299        // exercised in the pair kernel too.
8300        let mut x1: Vec<f32> = (0..cols).map(|i| (i as f32 * 0.17).sin()).collect();
8301        x1[9] = 250.0;
8302        let x2: Vec<f32> = (0..cols).map(|i| (i as f32 * 0.23).cos()).collect();
8303
8304        let (mut a1, mut a2) = (vec![0f32; rows], vec![0f32; rows]);
8305        q4matvec(&bytes, &x1, rows, cols, &mut a1, None);
8306        q4matvec(&bytes, &x2, rows, cols, &mut a2, None);
8307        let (mut b1, mut b2) = (vec![0f32; rows], vec![0f32; rows]);
8308        q4matvec2(&bytes, &x1, &x2, rows, cols, &mut b1, &mut b2, None);
8309        assert_eq!(a1, b1, "fused q4 lane 1 must be bit-identical");
8310        assert_eq!(a2, b2, "fused q4 lane 2 must be bit-identical");
8311    }
8312
8313    /// Multi-matrix job must equal separate matvecs exactly — same
8314    /// kernels, only the dispatch is fused.
8315    #[test]
8316    fn matvec_many_equals_separate_matvecs() {
8317        use crate::pool::Pool;
8318        let (r1, r2, cols) = (300, 200, 64);
8319        let mk = |salt: usize, rows: usize| {
8320            QTensor::from_f32(
8321                (0..rows * cols)
8322                    .map(|i| ((i * 7 + salt) % 97) as f32 / 97.0 - 0.5)
8323                    .collect(),
8324                rows,
8325                cols,
8326            )
8327        };
8328        let (a, b) = (mk(1, r1), mk(5, r2));
8329        let x: Vec<f32> = (0..cols).map(|i| (i as f32 * 0.11).sin()).collect();
8330        let pool = Pool::new(3);
8331
8332        let (mut ea, mut eb) = (vec![0f32; r1], vec![0f32; r2]);
8333        a.matvec(&x, &mut ea, Some(&pool));
8334        b.matvec(&x, &mut eb, Some(&pool));
8335        let (mut ga, mut gb) = (vec![0f32; r1], vec![0f32; r2]);
8336        QTensor::matvec_many([&a, &b], &x, [&mut ga, &mut gb], Some(&pool));
8337        assert_eq!(ea, ga, "fused multi-matrix lane 1 must be bit-identical");
8338        assert_eq!(eb, gb, "fused multi-matrix lane 2 must be bit-identical");
8339    }
8340
8341    /// Batched q4/vbit matmat must equal per-position matvec calls
8342    /// exactly (the fallback it replaced) — same kernels, same order.
8343    #[test]
8344    fn batched_matmat_equals_per_position_matvec() {
8345        let (rows, cols, b) = (8, 64, 5);
8346        // q4 blob.
8347        let groups = rows * cols / GROUP_SIZE;
8348        let mut q4 = Vec::new();
8349        for i in 0..groups * 16 {
8350            q4.push((((i * 7 + 3) % 256) & 0xFF) as u8);
8351        }
8352        for g in 0..groups {
8353            q4.extend_from_slice(
8354                &cortiq_core::quant::f32_to_f16(0.01 + 0.003 * g as f32).to_le_bytes(),
8355            );
8356        }
8357        // vbit blob (mixed widths incl. 8).
8358        let ng = cols / GROUP_SIZE;
8359        let bits: Vec<u8> = vec![3, 4, 5, 6, 8, 4, 5, 3];
8360        let mut vb = bits.clone();
8361        for g in 0..rows * ng {
8362            vb.extend_from_slice(
8363                &cortiq_core::quant::f32_to_f16(0.02 + 0.001 * g as f32).to_le_bytes(),
8364            );
8365        }
8366        for r in 0..rows {
8367            let bw = bits[r] as usize;
8368            let (mut acc, mut nb) = (0u64, 0usize);
8369            let mut rowbytes = Vec::new();
8370            for i in 0..cols {
8371                let v = ((i * 7 + r * 13) % (1 << bw)) as u64;
8372                acc = (acc << bw) | v;
8373                nb += bw;
8374                while nb >= 8 {
8375                    nb -= 8;
8376                    rowbytes.push(((acc >> nb) & 0xFF) as u8);
8377                }
8378            }
8379            if nb > 0 {
8380                rowbytes.push(((acc << (8 - nb)) & 0xFF) as u8);
8381            }
8382            vb.extend_from_slice(&rowbytes);
8383        }
8384        let offsets = vbit_row_offsets(&vb, rows, cols);
8385
8386        let xs: Vec<f32> = (0..b * cols).map(|i| (i as f32 * 0.13).sin()).collect();
8387
8388        // q4: batch vs singles.
8389        let mut got = vec![0f32; b * rows];
8390        q4matmat(&q4, &xs, b, rows, cols, &mut got, None);
8391        for bi in 0..b {
8392            let mut expect = vec![0f32; rows];
8393            q4matvec(
8394                &q4,
8395                &xs[bi * cols..(bi + 1) * cols],
8396                rows,
8397                cols,
8398                &mut expect,
8399                None,
8400            );
8401            assert_eq!(
8402                &got[bi * rows..(bi + 1) * rows],
8403                &expect[..],
8404                "q4 batch pos {bi}"
8405            );
8406        }
8407
8408        // vbit: batch vs singles.
8409        let mut got = vec![0f32; b * rows];
8410        vbitmatmat(&vb, &offsets, &xs, b, rows, cols, &mut got, None);
8411        for bi in 0..b {
8412            let mut expect = vec![0f32; rows];
8413            vbitmatvec(
8414                &vb,
8415                &offsets,
8416                &xs[bi * cols..(bi + 1) * cols],
8417                rows,
8418                cols,
8419                &mut expect,
8420                None,
8421            );
8422            assert_eq!(
8423                &got[bi * rows..(bi + 1) * rows],
8424                &expect[..],
8425                "vbit batch pos {bi}"
8426            );
8427        }
8428    }
8429
8430    /// q4_tiled kernels must produce BIT-identical outputs to the q4
8431    /// split kernels on the same values (same ints, same order — only
8432    /// the byte placement differs).
8433    #[test]
8434    fn q4_tiled_matches_q4_block_bitexact() {
8435        let (rows, cols, b) = (8usize, 128usize, 3usize);
8436        let groups = rows * cols / GROUP_SIZE;
8437        let mut split = Vec::with_capacity(groups * 18);
8438        for i in 0..groups * 16 {
8439            split.push((((i * 7 + 3) % 256) & 0xFF) as u8);
8440        }
8441        for g in 0..groups {
8442            split.extend_from_slice(
8443                &cortiq_core::quant::f32_to_f16(0.01 + 0.003 * g as f32).to_le_bytes(),
8444            );
8445        }
8446        // Re-tile: [scale][nibbles] per group.
8447        let (packed, scales) = split.split_at(groups * 16);
8448        let mut tiled = Vec::with_capacity(groups * Q4_TILE);
8449        for g in 0..groups {
8450            tiled.extend_from_slice(&scales[g * 2..g * 2 + 2]);
8451            tiled.extend_from_slice(&packed[g * 16..(g + 1) * 16]);
8452        }
8453
8454        let mut x1: Vec<f32> = (0..cols).map(|i| (i as f32 * 0.17).sin()).collect();
8455        x1[9] = 250.0; // exercise the outlier path
8456        let x2: Vec<f32> = (0..cols).map(|i| (i as f32 * 0.23).cos()).collect();
8457
8458        let (mut a, mut t) = (vec![0f32; rows], vec![0f32; rows]);
8459        q4matvec(&split, &x1, rows, cols, &mut a, None);
8460        q4t_matvec(&tiled, &x1, rows, cols, &mut t, None);
8461        assert_eq!(a, t, "q4t matvec must match q4 bit-for-bit");
8462
8463        let (mut a1, mut a2) = (vec![0f32; rows], vec![0f32; rows]);
8464        let (mut t1, mut t2) = (vec![0f32; rows], vec![0f32; rows]);
8465        q4matvec2(&split, &x1, &x2, rows, cols, &mut a1, &mut a2, None);
8466        q4t_matvec2(&tiled, &x1, &x2, rows, cols, &mut t1, &mut t2, None);
8467        assert_eq!(a1, t1);
8468        assert_eq!(a2, t2);
8469
8470        let xs: Vec<f32> = (0..b * cols).map(|i| (i as f32 * 0.13).sin()).collect();
8471        let (mut am, mut tm) = (vec![0f32; b * rows], vec![0f32; b * rows]);
8472        q4matmat(&split, &xs, b, rows, cols, &mut am, None);
8473        q4t_matmat(&tiled, &xs, b, rows, cols, &mut tm, None);
8474        assert_eq!(am, tm, "q4t matmat must match q4 bit-for-bit");
8475    }
8476
8477    /// q4 SDOT outlier correction: a single huge activation channel
8478    /// (>8·rms → outlier, zeroed in xq) must still contribute its EXACT
8479    /// term. On-grid bulk (±1/0 → xq dequantizes exactly) isolates the
8480    /// correction from A8W8 noise. cols must exceed 64: at n=64 the
8481    /// 8·rms threshold equals sqrt(v²+rest) ≥ v, so a single outlier
8482    /// can never qualify (8² = n).
8483    #[test]
8484    fn q4matvec_sdot_outlier_exact() {
8485        let (rows, cols) = (4, 128);
8486        let groups = rows * cols / GROUP_SIZE;
8487        let mut bytes = Vec::with_capacity(groups * 16 + groups * 2);
8488        for i in 0..groups * 16 {
8489            bytes.push(((i * 11 + 5) % 256) as u8);
8490        }
8491        for g in 0..groups {
8492            let s = 0.02 + 0.002 * g as f32;
8493            bytes.extend_from_slice(&cortiq_core::quant::f32_to_f16(s).to_le_bytes());
8494        }
8495        let mut x: Vec<f32> = (0..cols)
8496            .map(|i| match i % 3 {
8497                0 => 1.0,
8498                1 => -1.0,
8499                _ => 0.0,
8500            })
8501            .collect();
8502        x[17] = 300.0; // ≫ 8·rms → outlier channel
8503
8504        let mut reference = vec![0.0f32; rows * cols];
8505        cortiq_core::quant::dequant_q4_block(&bytes, &mut reference);
8506        let mut expect = vec![0.0f32; rows];
8507        for r in 0..rows {
8508            expect[r] = reference[r * cols..(r + 1) * cols]
8509                .iter()
8510                .zip(&x)
8511                .map(|(w, xv)| w * xv)
8512                .sum();
8513        }
8514        let mut got = vec![0.0f32; rows];
8515        q4matvec(&bytes, &x, rows, cols, &mut got, None);
8516        let scale = expect.iter().fold(0f32, |m, v| m.max(v.abs())).max(1.0);
8517        for r in 0..rows {
8518            assert!(
8519                (got[r] - expect[r]).abs() < 2e-3 * scale,
8520                "row {r}: {} vs {} (outlier term must be exact)",
8521                got[r],
8522                expect[r]
8523            );
8524        }
8525    }
8526
8527    /// The fused q1t matvec must equal the reference (dequant_q1t → dot),
8528    /// including the ternary zero level and the binary-searched outlier
8529    /// overlay. Guards the mmap kernel that makes a 12B q1t runnable.
8530    #[test]
8531    fn q1t_matvec_matches_reference() {
8532        use cortiq_core::quant::{dequant_q1t, f32_to_f16};
8533        let (rows, cols) = (3usize, 64usize); // gpr = 2
8534        let gpr = cols / GROUP_SIZE;
8535        let scales = [0.5f32, 0.3, 0.7, 0.2, 0.6, 0.15];
8536        // Overlay (must be sorted by flat index): a few spikes across rows.
8537        let outliers: [(u32, f32); 3] = [(5, 9.0), (70, -4.5), (150, 3.25)];
8538        let is_out = |flat: usize| outliers.iter().any(|&(i, _)| i as usize == flat);
8539        let mut bytes = Vec::new();
8540        for r in 0..rows {
8541            for g in 0..gpr {
8542                bytes.extend_from_slice(&f32_to_f16(scales[r * gpr + g]).to_le_bytes());
8543                let mut c = [0u8; 7];
8544                for k in 0..GROUP_SIZE {
8545                    // Encoder invariant: code 0 at outlier positions.
8546                    let code = if is_out(r * cols + g * GROUP_SIZE + k) {
8547                        0
8548                    } else {
8549                        ((k + r * 3 + g) % 3) as u8 // 0,1,2
8550                    };
8551                    cortiq_core::quant::q1t_pack(&mut c, k, code);
8552                }
8553                bytes.extend_from_slice(&c);
8554            }
8555        }
8556        // Per-row overlay: [u32 row_ptr[rows+1]] then [(u16 col, f16 val)] by
8557        // row (outliers are sorted by flat index → already grouped by row).
8558        let mut row_ptr = vec![0u32; rows + 1];
8559        for &(idx, _) in &outliers {
8560            row_ptr[idx as usize / cols + 1] += 1;
8561        }
8562        for r in 0..rows {
8563            row_ptr[r + 1] += row_ptr[r];
8564        }
8565        for &p in &row_ptr {
8566            bytes.extend_from_slice(&p.to_le_bytes());
8567        }
8568        for &(idx, v) in &outliers {
8569            bytes.extend_from_slice(&((idx as usize % cols) as u16).to_le_bytes());
8570            bytes.extend_from_slice(&f32_to_f16(v).to_le_bytes());
8571        }
8572
8573        let mut refw = vec![0f32; rows * cols];
8574        dequant_q1t(&bytes, rows, cols, &mut refw);
8575        // On-grid activations (±1, amax 1) so the int8 SDOT path reconstructs
8576        // x exactly and matches the f32 reference (same trick as the q1 test).
8577        let x: Vec<f32> = (0..cols)
8578            .map(|j| if j % 3 == 0 { 1.0 } else { -1.0 })
8579            .collect();
8580        let mut expect = vec![0f32; rows];
8581        for r in 0..rows {
8582            let mut a = 0.0f32;
8583            for j in 0..cols {
8584                a += refw[r * cols + j] * x[j];
8585            }
8586            expect[r] = a;
8587        }
8588        let tol = |e: f32| 1e-3 * e.abs().max(1e-3);
8589        let mut got = vec![0f32; rows];
8590        q1t_matvec(&bytes, &x, rows, cols, &mut got, None);
8591        for r in 0..rows {
8592            assert!(
8593                (got[r] - expect[r]).abs() < tol(expect[r]),
8594                "row {r}: {} vs {}",
8595                got[r],
8596                expect[r]
8597            );
8598        }
8599        // matmat (b=2, f32 decode path) must agree too.
8600        let x2: Vec<f32> = x.iter().chain(x.iter().map(|v| v)).copied().collect();
8601        let mut gm = vec![0f32; 2 * rows];
8602        q1t_matmat(&bytes, &x2, 2, rows, cols, &mut gm, None);
8603        for r in 0..rows {
8604            assert!((gm[r] - expect[r]).abs() < tol(expect[r]));
8605            assert!((gm[rows + r] - expect[r]).abs() < tol(expect[r]));
8606        }
8607        // Fused pair (q1t_matvec2) must equal two single matvecs
8608        // bit-for-bit: same unpack, same group order, same f32
8609        // accumulation per stream. Distinct x2 exercises both lanes.
8610        let xb: Vec<f32> = (0..cols)
8611            .map(|j| if j % 5 == 0 { -1.0 } else { 1.0 })
8612            .collect();
8613        let (mut s1, mut s2) = (vec![0f32; rows], vec![0f32; rows]);
8614        q1t_matvec(&bytes, &x, rows, cols, &mut s1, None);
8615        q1t_matvec(&bytes, &xb, rows, cols, &mut s2, None);
8616        let (mut p1, mut p2) = (vec![0f32; rows], vec![0f32; rows]);
8617        q1t_matvec2(&bytes, &x, &xb, rows, cols, &mut p1, &mut p2, None);
8618        assert_eq!(p1, s1, "q1t pair lane 1 ≠ single matvec");
8619        assert_eq!(p2, s2, "q1t pair lane 2 ≠ single matvec");
8620    }
8621
8622    /// Pair == 2×matvec with an ODD group count (the kernel's tail
8623    /// group) and no overlay section.
8624    #[test]
8625    fn q1t_matvec2_odd_gpr_matches_singles() {
8626        use cortiq_core::quant::{Q1T_TILE, f32_to_f16, q1t_pack};
8627        let (rows, cols) = (5usize, 96usize); // gpr = 3 → paired + tail
8628        let gpr = cols / GROUP_SIZE;
8629        let mut bytes = Vec::with_capacity(rows * gpr * Q1T_TILE);
8630        for r in 0..rows {
8631            for g in 0..gpr {
8632                bytes.extend_from_slice(&f32_to_f16(0.1 + 0.05 * (r + g) as f32).to_le_bytes());
8633                let mut c = [0u8; 7];
8634                for k in 0..GROUP_SIZE {
8635                    q1t_pack(&mut c, k, ((k * 7 + r * 5 + g * 3) % 3) as u8);
8636                }
8637                bytes.extend_from_slice(&c);
8638            }
8639        }
8640        let x1: Vec<f32> = (0..cols).map(|i| (i as f32 * 0.31).sin()).collect();
8641        let x2: Vec<f32> = (0..cols).map(|i| (i as f32 * 0.17).cos()).collect();
8642        let (mut s1, mut s2) = (vec![0f32; rows], vec![0f32; rows]);
8643        q1t_matvec(&bytes, &x1, rows, cols, &mut s1, None);
8644        q1t_matvec(&bytes, &x2, rows, cols, &mut s2, None);
8645        let (mut p1, mut p2) = (vec![0f32; rows], vec![0f32; rows]);
8646        q1t_matvec2(&bytes, &x1, &x2, rows, cols, &mut p1, &mut p2, None);
8647        assert_eq!(p1, s1, "odd-gpr pair lane 1 ≠ single");
8648        assert_eq!(p2, s2, "odd-gpr pair lane 2 ≠ single");
8649    }
8650
8651    // Speed A/B: fused pair (one unpack, two streams) vs two single
8652    // matvecs. Single-threaded, FFN-sized, min-of paired in-process.
8653    //   cargo test -p cortiq-engine --release q1t_matvec2_speed -- --ignored --nocapture
8654    #[test]
8655    #[ignore]
8656    fn q1t_matvec2_speed() {
8657        use cortiq_core::quant::{Q1T_TILE, f32_to_f16, q1t_pack};
8658        use std::time::Instant;
8659        let (rows, cols) = (8192usize, 4096usize);
8660        let gpr = cols / GROUP_SIZE;
8661        let mut bytes = Vec::with_capacity(rows * gpr * Q1T_TILE);
8662        for r in 0..rows {
8663            for g in 0..gpr {
8664                let s = 0.1 + ((r + g) % 7) as f32 * 0.01;
8665                bytes.extend_from_slice(&f32_to_f16(s).to_le_bytes());
8666                let mut c = [0u8; 7];
8667                for k in 0..GROUP_SIZE {
8668                    q1t_pack(&mut c, k, ((k * 7 + r + g) % 3) as u8);
8669                }
8670                bytes.extend_from_slice(&c);
8671            }
8672        }
8673        let x1: Vec<f32> = (0..cols).map(|i| (i as f32 * 0.31).sin()).collect();
8674        let x2: Vec<f32> = (0..cols).map(|i| (i as f32 * 0.17).cos()).collect();
8675        let (mut s1, mut s2) = (vec![0f32; rows], vec![0f32; rows]);
8676        let (mut p1, mut p2) = (vec![0f32; rows], vec![0f32; rows]);
8677        // Warm both paths once.
8678        q1t_matvec(&bytes, &x1, rows, cols, &mut s1, None);
8679        q1t_matvec2(&bytes, &x1, &x2, rows, cols, &mut p1, &mut p2, None);
8680        let (mut t_pair, mut t_two) = (f64::MAX, f64::MAX);
8681        for _ in 0..8 {
8682            let t0 = Instant::now();
8683            q1t_matvec2(&bytes, &x1, &x2, rows, cols, &mut p1, &mut p2, None);
8684            t_pair = t_pair.min(t0.elapsed().as_secs_f64() * 1000.0);
8685            let t1 = Instant::now();
8686            q1t_matvec(&bytes, &x1, rows, cols, &mut s1, None);
8687            q1t_matvec(&bytes, &x2, rows, cols, &mut s2, None);
8688            t_two = t_two.min(t1.elapsed().as_secs_f64() * 1000.0);
8689        }
8690        assert_eq!(p1, s1);
8691        assert_eq!(p2, s2);
8692        println!("q1t pair {rows}x{cols}: fused {t_pair:.2} ms | two singles {t_two:.2} ms");
8693    }
8694
8695    // Speed A/B: the base-3-division decode (what the packing commit left in
8696    // place) vs the fused sign-LUT matvec. Both single-threaded, same bytes.
8697    //   cargo test -p cortiq-engine q1t_matvec_speed -- --ignored --nocapture
8698    #[test]
8699    #[ignore]
8700    fn q1t_matvec_speed() {
8701        use cortiq_core::quant::{Q1T_TILE, f32_to_f16, q1t_code, q1t_pack};
8702        use std::time::Instant;
8703        let (rows, cols) = (8192usize, 4096usize); // FFN-sized
8704        let gpr = cols / GROUP_SIZE;
8705        let mut bytes = Vec::with_capacity(rows * gpr * Q1T_TILE + 16);
8706        for r in 0..rows {
8707            for g in 0..gpr {
8708                let s = 0.1 + ((r + g) % 7) as f32 * 0.01;
8709                bytes.extend_from_slice(&f32_to_f16(s).to_le_bytes());
8710                let mut c = [0u8; 7];
8711                for k in 0..GROUP_SIZE {
8712                    q1t_pack(&mut c, k, ((k * 7 + r + g) % 3) as u8);
8713                }
8714                bytes.extend_from_slice(&c);
8715            }
8716        }
8717        let (n, stride) = (rows * cols, 40usize); // ~2.5% outliers, per-row overlay
8718        let mut row_ptr = vec![0u32; rows + 1];
8719        let mut idx = 0usize;
8720        while idx < n {
8721            row_ptr[idx / cols + 1] += 1;
8722            idx += stride;
8723        }
8724        for r in 0..rows {
8725            row_ptr[r + 1] += row_ptr[r];
8726        }
8727        for &p in &row_ptr {
8728            bytes.extend_from_slice(&p.to_le_bytes());
8729        }
8730        let mut idx = 0usize;
8731        while idx < n {
8732            bytes.extend_from_slice(&((idx % cols) as u16).to_le_bytes());
8733            bytes.extend_from_slice(&f32_to_f16((idx % 13) as f32 * 0.1 - 0.6).to_le_bytes());
8734            idx += stride;
8735        }
8736        // On-grid ±1 so the fast path's int8 SDOT is exact vs the f32 "slow"
8737        // reference (the A/B is a timing check; values must still agree).
8738        let x: Vec<f32> = (0..cols)
8739            .map(|j| if j % 3 == 0 { 1.0 } else { -1.0 })
8740            .collect();
8741        let (rp_off, ent_off, has_ov) = q1t_overlay(&bytes, rows * gpr * Q1T_TILE, rows);
8742
8743        // "before": base-3 division decode into a buffer, then dot.
8744        let slow = |out: &mut [f32]| {
8745            let mut buf = vec![0f32; cols];
8746            for r in 0..rows {
8747                for g in 0..gpr {
8748                    let off = (r * gpr + g) * Q1T_TILE;
8749                    let s = f16_to_f32(u16::from_le_bytes([bytes[off], bytes[off + 1]]));
8750                    let codes = &bytes[off + 2..off + Q1T_TILE];
8751                    for k in 0..GROUP_SIZE {
8752                        buf[g * GROUP_SIZE + k] = match q1t_code(codes, k) {
8753                            1 => s,
8754                            2 => -s,
8755                            _ => 0.0,
8756                        };
8757                    }
8758                }
8759                out[r] = q1t_row_outlier_correction(&bytes, r, rp_off, ent_off, has_ov, &x)
8760                    + (0..cols).map(|j| buf[j] * x[j]).sum::<f32>();
8761            }
8762        };
8763        let iters = 5;
8764        let mut a = vec![0f32; rows];
8765        slow(&mut a); // warm
8766        let t = Instant::now();
8767        for _ in 0..iters {
8768            slow(&mut a);
8769        }
8770        let slow_ms = t.elapsed().as_secs_f64() * 1e3 / iters as f64;
8771
8772        let mut b = vec![0f32; rows];
8773        q1t_matvec(&bytes, &x, rows, cols, &mut b, None); // warm
8774        let t = Instant::now();
8775        for _ in 0..iters {
8776            q1t_matvec(&bytes, &x, rows, cols, &mut b, None);
8777        }
8778        let fast_ms = t.elapsed().as_secs_f64() * 1e3 / iters as f64;
8779
8780        for r in 0..rows {
8781            assert!((a[r] - b[r]).abs() < 1e-2, "mismatch row {r}");
8782        }
8783        println!(
8784            "q1t matvec {rows}x{cols} (1 thread): div-decode {slow_ms:.2} ms  fused-LUT {fast_ms:.2} ms  => {:.2}x",
8785            slow_ms / fast_ms
8786        );
8787    }
8788}
cortiq_engine/qtensor.rs

cortiq_engine/
qtensor.rs
