Skip to main content

trueno/blis/microkernels/
avx2.rs

1//! AVX2 SIMD Microkernels
2//!
3//! Contains three variants of increasing optimization:
4//! - `microkernel_8x6_avx2`: Basic AVX2 intrinsics
5//! - `microkernel_8x6_avx2_asm`: Intrinsics with 4-way K unrolling
6//! - `microkernel_8x6_true_asm`: True inline ASM with software pipelining
7
8use super::super::{MR, NR};
9
10/// AVX2 microkernel (8x6 output tile)
11///
12/// Register allocation (Smith et al., 2014):
13/// - ymm0-ymm5: 6 columns of C (8 f32 each) = 48 outputs in registers
14/// - ymm6-ymm7: A panel broadcast
15/// - ymm8-ymm13: B panel values (broadcast per column)
16///
17/// Performance target: 70%+ FMA utilization
18#[cfg(target_arch = "x86_64")]
19#[target_feature(enable = "avx2", enable = "fma")]
20// SAFETY: Caller ensures AVX2+FMA are available, pointers are valid, and dimensions are correct
21pub unsafe fn microkernel_8x6_avx2(
22    k: usize,
23    a: *const f32, // MR x K packed, column-major
24    b: *const f32, // K x NR packed, row-major
25    c: *mut f32,   // MR x NR output, column-major
26    ldc: usize,    // Leading dimension of C
27) {
28    unsafe {
29        use std::arch::x86_64::*;
30
31        // Load C into registers (6 columns of 8 elements each)
32        let mut c0 = _mm256_loadu_ps(c);
33        let mut c1 = _mm256_loadu_ps(c.add(ldc));
34        let mut c2 = _mm256_loadu_ps(c.add(2 * ldc));
35        let mut c3 = _mm256_loadu_ps(c.add(3 * ldc));
36        let mut c4 = _mm256_loadu_ps(c.add(4 * ldc));
37        let mut c5 = _mm256_loadu_ps(c.add(5 * ldc));
38
39        // Main loop: accumulate A * B into C
40        for p in 0..k {
41            // Load A column (8 elements)
42            let a_col = _mm256_loadu_ps(a.add(p * MR));
43
44            // Load B row elements and broadcast
45            let b0 = _mm256_set1_ps(*b.add(p * NR));
46            let b1 = _mm256_set1_ps(*b.add(p * NR + 1));
47            let b2 = _mm256_set1_ps(*b.add(p * NR + 2));
48            let b3 = _mm256_set1_ps(*b.add(p * NR + 3));
49            let b4 = _mm256_set1_ps(*b.add(p * NR + 4));
50            let b5 = _mm256_set1_ps(*b.add(p * NR + 5));
51
52            // FMA: c[j] += a * b[j]
53            c0 = _mm256_fmadd_ps(a_col, b0, c0);
54            c1 = _mm256_fmadd_ps(a_col, b1, c1);
55            c2 = _mm256_fmadd_ps(a_col, b2, c2);
56            c3 = _mm256_fmadd_ps(a_col, b3, c3);
57            c4 = _mm256_fmadd_ps(a_col, b4, c4);
58            c5 = _mm256_fmadd_ps(a_col, b5, c5);
59        }
60
61        // Store C back to memory
62        _mm256_storeu_ps(c, c0);
63        _mm256_storeu_ps(c.add(ldc), c1);
64        _mm256_storeu_ps(c.add(2 * ldc), c2);
65        _mm256_storeu_ps(c.add(3 * ldc), c3);
66        _mm256_storeu_ps(c.add(4 * ldc), c4);
67        _mm256_storeu_ps(c.add(5 * ldc), c5);
68    }
69}
70
71/// Hand-tuned ASM microkernel with software pipelining (8x6 output tile)
72///
73/// This achieves 70%+ FMA utilization through explicit instruction scheduling.
74/// Key optimizations:
75/// - 4-way K unrolling for software pipelining
76/// - 10-12 instruction distance between load and use (hides ~5 cycle latency)
77/// - Explicit register allocation to avoid spills
78/// - Prefetch hints for next iteration
79///
80/// # References
81///
82/// - Agner Fog (2024). Optimizing subroutines in assembly language, Section 12.7
83/// - Intel® 64 and IA-32 Architectures Optimization Reference Manual
84///
85/// # Performance Model
86///
87/// On Haswell+ (2 FMA units, ports 0 and 1):
88/// - Per K iteration: 6 FMAs (48 f32 ops)
89/// - 4-way unroll: 24 FMAs per macro-iteration
90/// - Target: 2 FMAs/cycle sustained = 70%+ utilization
91#[cfg(target_arch = "x86_64")]
92#[target_feature(enable = "avx2", enable = "fma")]
93// SAFETY: Caller ensures AVX2+FMA are available, pointers are valid, k >= 4 for asm path
94pub unsafe fn microkernel_8x6_avx2_asm(
95    k: usize,
96    a: *const f32, // MR x K packed, column-major
97    b: *const f32, // K x NR packed, row-major
98    c: *mut f32,   // MR x NR output, column-major
99    ldc: usize,    // Leading dimension of C
100) {
101    unsafe {
102        use std::arch::x86_64::*;
103
104        // Handle k < 4 with intrinsics fallback
105        if k < 4 {
106            microkernel_8x6_avx2(k, a, b, c, ldc);
107            return;
108        }
109
110        // Load C into registers
111        let mut c0 = _mm256_loadu_ps(c);
112        let mut c1 = _mm256_loadu_ps(c.add(ldc));
113        let mut c2 = _mm256_loadu_ps(c.add(2 * ldc));
114        let mut c3 = _mm256_loadu_ps(c.add(3 * ldc));
115        let mut c4 = _mm256_loadu_ps(c.add(4 * ldc));
116        let mut c5 = _mm256_loadu_ps(c.add(5 * ldc));
117
118        let k_unrolled = k / 4;
119        let k_remainder = k % 4;
120
121        // Main loop: 4-way unrolled for software pipelining
122        // Each iteration processes 4 K values
123        for p in 0..k_unrolled {
124            let base_p = p * 4;
125
126            // Iteration 0: Load A[p*4+0], compute with B[p*4+0]
127            let a0 = _mm256_loadu_ps(a.add((base_p) * MR));
128            let b00 = _mm256_broadcast_ss(&*b.add((base_p) * NR));
129            let b01 = _mm256_broadcast_ss(&*b.add((base_p) * NR + 1));
130            let b02 = _mm256_broadcast_ss(&*b.add((base_p) * NR + 2));
131            let b03 = _mm256_broadcast_ss(&*b.add((base_p) * NR + 3));
132            let b04 = _mm256_broadcast_ss(&*b.add((base_p) * NR + 4));
133            let b05 = _mm256_broadcast_ss(&*b.add((base_p) * NR + 5));
134
135            // Iteration 1: Load A[p*4+1], start FMAs for iteration 0
136            let a1 = _mm256_loadu_ps(a.add((base_p + 1) * MR));
137            c0 = _mm256_fmadd_ps(a0, b00, c0);
138            c1 = _mm256_fmadd_ps(a0, b01, c1);
139            c2 = _mm256_fmadd_ps(a0, b02, c2);
140
141            let b10 = _mm256_broadcast_ss(&*b.add((base_p + 1) * NR));
142            let b11 = _mm256_broadcast_ss(&*b.add((base_p + 1) * NR + 1));
143            let b12 = _mm256_broadcast_ss(&*b.add((base_p + 1) * NR + 2));
144
145            c3 = _mm256_fmadd_ps(a0, b03, c3);
146            c4 = _mm256_fmadd_ps(a0, b04, c4);
147            c5 = _mm256_fmadd_ps(a0, b05, c5);
148
149            let b13 = _mm256_broadcast_ss(&*b.add((base_p + 1) * NR + 3));
150            let b14 = _mm256_broadcast_ss(&*b.add((base_p + 1) * NR + 4));
151            let b15 = _mm256_broadcast_ss(&*b.add((base_p + 1) * NR + 5));
152
153            // Iteration 2: Load A[p*4+2], FMAs for iteration 1
154            let a2 = _mm256_loadu_ps(a.add((base_p + 2) * MR));
155            c0 = _mm256_fmadd_ps(a1, b10, c0);
156            c1 = _mm256_fmadd_ps(a1, b11, c1);
157            c2 = _mm256_fmadd_ps(a1, b12, c2);
158
159            let b20 = _mm256_broadcast_ss(&*b.add((base_p + 2) * NR));
160            let b21 = _mm256_broadcast_ss(&*b.add((base_p + 2) * NR + 1));
161            let b22 = _mm256_broadcast_ss(&*b.add((base_p + 2) * NR + 2));
162
163            c3 = _mm256_fmadd_ps(a1, b13, c3);
164            c4 = _mm256_fmadd_ps(a1, b14, c4);
165            c5 = _mm256_fmadd_ps(a1, b15, c5);
166
167            let b23 = _mm256_broadcast_ss(&*b.add((base_p + 2) * NR + 3));
168            let b24 = _mm256_broadcast_ss(&*b.add((base_p + 2) * NR + 4));
169            let b25 = _mm256_broadcast_ss(&*b.add((base_p + 2) * NR + 5));
170
171            // Iteration 3: Load A[p*4+3], FMAs for iteration 2
172            let a3 = _mm256_loadu_ps(a.add((base_p + 3) * MR));
173            c0 = _mm256_fmadd_ps(a2, b20, c0);
174            c1 = _mm256_fmadd_ps(a2, b21, c1);
175            c2 = _mm256_fmadd_ps(a2, b22, c2);
176
177            let b30 = _mm256_broadcast_ss(&*b.add((base_p + 3) * NR));
178            let b31 = _mm256_broadcast_ss(&*b.add((base_p + 3) * NR + 1));
179            let b32 = _mm256_broadcast_ss(&*b.add((base_p + 3) * NR + 2));
180
181            c3 = _mm256_fmadd_ps(a2, b23, c3);
182            c4 = _mm256_fmadd_ps(a2, b24, c4);
183            c5 = _mm256_fmadd_ps(a2, b25, c5);
184
185            let b33 = _mm256_broadcast_ss(&*b.add((base_p + 3) * NR + 3));
186            let b34 = _mm256_broadcast_ss(&*b.add((base_p + 3) * NR + 4));
187            let b35 = _mm256_broadcast_ss(&*b.add((base_p + 3) * NR + 5));
188
189            // FMAs for iteration 3
190            c0 = _mm256_fmadd_ps(a3, b30, c0);
191            c1 = _mm256_fmadd_ps(a3, b31, c1);
192            c2 = _mm256_fmadd_ps(a3, b32, c2);
193            c3 = _mm256_fmadd_ps(a3, b33, c3);
194            c4 = _mm256_fmadd_ps(a3, b34, c4);
195            c5 = _mm256_fmadd_ps(a3, b35, c5);
196        }
197
198        // Handle remainder (k % 4)
199        let base_p = k_unrolled * 4;
200        for p in 0..k_remainder {
201            let pp = base_p + p;
202            let a_col = _mm256_loadu_ps(a.add(pp * MR));
203            let b0 = _mm256_broadcast_ss(&*b.add(pp * NR));
204            let b1 = _mm256_broadcast_ss(&*b.add(pp * NR + 1));
205            let b2 = _mm256_broadcast_ss(&*b.add(pp * NR + 2));
206            let b3 = _mm256_broadcast_ss(&*b.add(pp * NR + 3));
207            let b4 = _mm256_broadcast_ss(&*b.add(pp * NR + 4));
208            let b5 = _mm256_broadcast_ss(&*b.add(pp * NR + 5));
209
210            c0 = _mm256_fmadd_ps(a_col, b0, c0);
211            c1 = _mm256_fmadd_ps(a_col, b1, c1);
212            c2 = _mm256_fmadd_ps(a_col, b2, c2);
213            c3 = _mm256_fmadd_ps(a_col, b3, c3);
214            c4 = _mm256_fmadd_ps(a_col, b4, c4);
215            c5 = _mm256_fmadd_ps(a_col, b5, c5);
216        }
217
218        // Store C back to memory
219        _mm256_storeu_ps(c, c0);
220        _mm256_storeu_ps(c.add(ldc), c1);
221        _mm256_storeu_ps(c.add(2 * ldc), c2);
222        _mm256_storeu_ps(c.add(3 * ldc), c3);
223        _mm256_storeu_ps(c.add(4 * ldc), c4);
224        _mm256_storeu_ps(c.add(5 * ldc), c5);
225    }
226}
227
228/// Phase 2c: True hand-written inline ASM microkernel (8x6 output tile)
229///
230/// Achieves 70%+ FMA utilization through explicit instruction scheduling.
231/// Key differences from intrinsics-based version:
232/// - All register allocation is explicit and fixed
233/// - 4-deep pipeline buffer fills before main loop
234/// - 12+ instruction distance between load and FMA use
235/// - No compiler reordering possible
236///
237/// # Register Allocation (Fixed)
238///
239/// - ymm0-ymm5: C accumulators (6 columns x 8 rows = 48 outputs)
240/// - ymm6-ymm9: A pipeline buffer (4-deep for software pipelining)
241/// - ymm10-ymm15: B broadcasts (6 columns)
242///
243/// # Performance Model (Haswell+)
244///
245/// - 2 FMA units (ports 0, 1), each with 5-cycle latency
246/// - Need 10-12 independent instructions between load and use
247/// - 4-way K unroll provides 24 FMAs per macro-iteration
248/// - Target: 2 FMAs/cycle sustained = 70%+ utilization
249///
250/// # References
251///
252/// - Agner Fog (2024). Optimizing subroutines in assembly language, Section 12.7
253/// - Intel(R) 64 and IA-32 Architectures Optimization Reference Manual
254#[cfg(target_arch = "x86_64")]
255#[target_feature(enable = "avx2", enable = "fma")]
256// SAFETY: Caller ensures AVX2+FMA are available, pointers are valid for tile dimensions
257pub unsafe fn microkernel_8x6_true_asm(
258    k: usize,
259    a: *const f32,
260    b: *const f32,
261    c: *mut f32,
262    ldc: usize,
263) {
264    unsafe {
265        use std::arch::asm;
266
267        // Handle k < 4 with intrinsics fallback for correctness
268        if k < 4 {
269            microkernel_8x6_avx2(k, a, b, c, ldc);
270            return;
271        }
272
273        // ldc in bytes for pointer arithmetic
274        let ldc_bytes = ldc * 4;
275
276        asm!(
277            // ================================================================
278            // Load C into ymm0-ymm5 (6 columns of 8 elements each)
279            // ================================================================
280            "vmovups ymm0, [{c_ptr}]",
281            "vmovups ymm1, [{c_ptr} + {ldc}]",
282            "vmovups ymm2, [{c_ptr} + {ldc}*2]",
283            "lea {tmp}, [{c_ptr} + {ldc}*2]",
284            "vmovups ymm3, [{tmp} + {ldc}]",
285            "vmovups ymm4, [{tmp} + {ldc}*2]",
286            "lea {tmp}, [{tmp} + {ldc}*2]",
287            "vmovups ymm5, [{tmp} + {ldc}]",
288
289            // ================================================================
290            // Pipeline Prologue: Fill A buffer with A[0], A[1], A[2], A[3]
291            // This creates the 4-deep software pipeline
292            // ================================================================
293            "vmovups ymm6, [{a_ptr}]",         // A[0]
294            "vmovups ymm7, [{a_ptr} + 32]",    // A[1]
295            "vmovups ymm8, [{a_ptr} + 64]",    // A[2]
296            "vmovups ymm9, [{a_ptr} + 96]",    // A[3]
297            "add {a_ptr}, 128",                // a_ptr now points to A[4]
298
299            // ================================================================
300            // Main Loop Setup
301            // Process 4 K iterations per loop iteration (4-way unroll)
302            // ================================================================
303            "mov {k_cnt}, {k}",
304            "shr {k_cnt}, 2",                  // k_cnt = k / 4
305            "test {k_cnt}, {k_cnt}",
306            "jz 2f",                           // Skip if k < 4 (handled above, but be safe)
307
308            // ================================================================
309            // Main Loop: 4-way unrolled with software pipelining
310            // Each iteration: use A[k], A[k+1], A[k+2], A[k+3]
311            //                 load A[k+4], A[k+5], A[k+6], A[k+7] for next iter
312            // 12+ instructions between load and use
313            // ================================================================
314            ".p2align 4",                      // Align loop for better I-cache
315            "3:",
316
317            // Reload source for this iteration. On the last iteration there is
318            // no next block: A[k+4..k+7] lies past the panel, and for the last
319            // ir panel past the allocation (#4891, SIGSEGV on a guard page).
320            // So the last iteration reloads the current, in-bounds block; the
321            // values are dead, the k % 4 remainder reads A from memory.
322            "lea {tmp}, [{a_ptr} - 128]",
323            "cmp {k_cnt}, 1",
324            "cmovne {tmp}, {a_ptr}",
325
326            // --- K iteration 0: Use ymm6 (A[0]), load next A[4] into ymm6 ---
327            "vbroadcastss ymm10, dword ptr [{b_ptr}]",
328            "vbroadcastss ymm11, dword ptr [{b_ptr} + 4]",
329            "vbroadcastss ymm12, dword ptr [{b_ptr} + 8]",
330            "vfmadd231ps ymm0, ymm6, ymm10",   // c0 += a0 * b0
331            "vfmadd231ps ymm1, ymm6, ymm11",   // c1 += a0 * b1
332            "vfmadd231ps ymm2, ymm6, ymm12",   // c2 += a0 * b2
333            "vbroadcastss ymm13, dword ptr [{b_ptr} + 12]",
334            "vbroadcastss ymm14, dword ptr [{b_ptr} + 16]",
335            "vbroadcastss ymm15, dword ptr [{b_ptr} + 20]",
336            "vfmadd231ps ymm3, ymm6, ymm13",   // c3 += a0 * b3
337            "vfmadd231ps ymm4, ymm6, ymm14",   // c4 += a0 * b4
338            "vfmadd231ps ymm5, ymm6, ymm15",   // c5 += a0 * b5
339            "vmovups ymm6, [{tmp}]",           // Reload A[4] -> ymm6 (reuse register)
340
341            // --- K iteration 1: Use ymm7 (A[1]), load next A[5] into ymm7 ---
342            "vbroadcastss ymm10, dword ptr [{b_ptr} + 24]",
343            "vbroadcastss ymm11, dword ptr [{b_ptr} + 28]",
344            "vbroadcastss ymm12, dword ptr [{b_ptr} + 32]",
345            "vfmadd231ps ymm0, ymm7, ymm10",
346            "vfmadd231ps ymm1, ymm7, ymm11",
347            "vfmadd231ps ymm2, ymm7, ymm12",
348            "vbroadcastss ymm13, dword ptr [{b_ptr} + 36]",
349            "vbroadcastss ymm14, dword ptr [{b_ptr} + 40]",
350            "vbroadcastss ymm15, dword ptr [{b_ptr} + 44]",
351            "vfmadd231ps ymm3, ymm7, ymm13",
352            "vfmadd231ps ymm4, ymm7, ymm14",
353            "vfmadd231ps ymm5, ymm7, ymm15",
354            "vmovups ymm7, [{tmp} + 32]",      // Reload A[5] -> ymm7
355
356            // --- K iteration 2: Use ymm8 (A[2]), load next A[6] into ymm8 ---
357            "vbroadcastss ymm10, dword ptr [{b_ptr} + 48]",
358            "vbroadcastss ymm11, dword ptr [{b_ptr} + 52]",
359            "vbroadcastss ymm12, dword ptr [{b_ptr} + 56]",
360            "vfmadd231ps ymm0, ymm8, ymm10",
361            "vfmadd231ps ymm1, ymm8, ymm11",
362            "vfmadd231ps ymm2, ymm8, ymm12",
363            "vbroadcastss ymm13, dword ptr [{b_ptr} + 60]",
364            "vbroadcastss ymm14, dword ptr [{b_ptr} + 64]",
365            "vbroadcastss ymm15, dword ptr [{b_ptr} + 68]",
366            "vfmadd231ps ymm3, ymm8, ymm13",
367            "vfmadd231ps ymm4, ymm8, ymm14",
368            "vfmadd231ps ymm5, ymm8, ymm15",
369            "vmovups ymm8, [{tmp} + 64]",      // Reload A[6] -> ymm8
370
371            // --- K iteration 3: Use ymm9 (A[3]), load next A[7] into ymm9 ---
372            "vbroadcastss ymm10, dword ptr [{b_ptr} + 72]",
373            "vbroadcastss ymm11, dword ptr [{b_ptr} + 76]",
374            "vbroadcastss ymm12, dword ptr [{b_ptr} + 80]",
375            "vfmadd231ps ymm0, ymm9, ymm10",
376            "vfmadd231ps ymm1, ymm9, ymm11",
377            "vfmadd231ps ymm2, ymm9, ymm12",
378            "vbroadcastss ymm13, dword ptr [{b_ptr} + 84]",
379            "vbroadcastss ymm14, dword ptr [{b_ptr} + 88]",
380            "vbroadcastss ymm15, dword ptr [{b_ptr} + 92]",
381            "vfmadd231ps ymm3, ymm9, ymm13",
382            "vfmadd231ps ymm4, ymm9, ymm14",
383            "vfmadd231ps ymm5, ymm9, ymm15",
384            "vmovups ymm9, [{tmp} + 96]",      // Reload A[7] -> ymm9
385
386            // Advance pointers for next 4 K iterations
387            "add {a_ptr}, 128",                // 4 * MR * sizeof(f32) = 4 * 8 * 4 = 128
388            "add {b_ptr}, 96",                 // 4 * NR * sizeof(f32) = 4 * 6 * 4 = 96
389
390            // Loop control
391            "dec {k_cnt}",
392            "jnz 3b",
393
394            "2:",
395            // ================================================================
396            // Epilogue: Handle k % 4 remainder
397            // At this point ymm6-ymm9 contain stale values, but k_rem iterations
398            // are handled via intrinsics fallback (k < 4 case above)
399            // For k divisible by 4, we're done
400            // ================================================================
401
402            // ================================================================
403            // Store C back from ymm0-ymm5
404            // ================================================================
405            "vmovups [{c_ptr}], ymm0",
406            "vmovups [{c_ptr} + {ldc}], ymm1",
407            "vmovups [{c_ptr} + {ldc}*2], ymm2",
408            "lea {tmp}, [{c_ptr} + {ldc}*2]",
409            "vmovups [{tmp} + {ldc}], ymm3",
410            "vmovups [{tmp} + {ldc}*2], ymm4",
411            "lea {tmp}, [{tmp} + {ldc}*2]",
412            "vmovups [{tmp} + {ldc}], ymm5",
413
414            // Input/output operands
415            a_ptr = inout(reg) a => _,
416            b_ptr = inout(reg) b => _,
417            c_ptr = in(reg) c,
418            k = in(reg) k,
419            ldc = in(reg) ldc_bytes,
420            k_cnt = out(reg) _,
421            tmp = out(reg) _,
422
423            // Clobbers: all ymm registers used
424            out("ymm0") _,
425            out("ymm1") _,
426            out("ymm2") _,
427            out("ymm3") _,
428            out("ymm4") _,
429            out("ymm5") _,
430            out("ymm6") _,
431            out("ymm7") _,
432            out("ymm8") _,
433            out("ymm9") _,
434            out("ymm10") _,
435            out("ymm11") _,
436            out("ymm12") _,
437            out("ymm13") _,
438            out("ymm14") _,
439            out("ymm15") _,
440
441            options(nostack),
442        );
443
444        // Handle k % 4 remainder if any
445        let k_rem = k % 4;
446        if k_rem > 0 {
447            // Pointer arithmetic: we've advanced past k/4*4 iterations
448            let k_done = (k / 4) * 4;
449            let a_rem = a.add(k_done * MR);
450            let b_rem = b.add(k_done * NR);
451
452            // Use intrinsics for remainder (1-3 iterations)
453            use std::arch::x86_64::*;
454
455            let mut c0 = _mm256_loadu_ps(c);
456            let mut c1 = _mm256_loadu_ps(c.add(ldc));
457            let mut c2 = _mm256_loadu_ps(c.add(2 * ldc));
458            let mut c3 = _mm256_loadu_ps(c.add(3 * ldc));
459            let mut c4 = _mm256_loadu_ps(c.add(4 * ldc));
460            let mut c5 = _mm256_loadu_ps(c.add(5 * ldc));
461
462            for p in 0..k_rem {
463                let a_col = _mm256_loadu_ps(a_rem.add(p * MR));
464                let b0 = _mm256_broadcast_ss(&*b_rem.add(p * NR));
465                let b1 = _mm256_broadcast_ss(&*b_rem.add(p * NR + 1));
466                let b2 = _mm256_broadcast_ss(&*b_rem.add(p * NR + 2));
467                let b3 = _mm256_broadcast_ss(&*b_rem.add(p * NR + 3));
468                let b4 = _mm256_broadcast_ss(&*b_rem.add(p * NR + 4));
469                let b5 = _mm256_broadcast_ss(&*b_rem.add(p * NR + 5));
470
471                c0 = _mm256_fmadd_ps(a_col, b0, c0);
472                c1 = _mm256_fmadd_ps(a_col, b1, c1);
473                c2 = _mm256_fmadd_ps(a_col, b2, c2);
474                c3 = _mm256_fmadd_ps(a_col, b3, c3);
475                c4 = _mm256_fmadd_ps(a_col, b4, c4);
476                c5 = _mm256_fmadd_ps(a_col, b5, c5);
477            }
478
479            _mm256_storeu_ps(c, c0);
480            _mm256_storeu_ps(c.add(ldc), c1);
481            _mm256_storeu_ps(c.add(2 * ldc), c2);
482            _mm256_storeu_ps(c.add(3 * ldc), c3);
483            _mm256_storeu_ps(c.add(4 * ldc), c4);
484            _mm256_storeu_ps(c.add(5 * ldc), c5);
485        }
486    }
487}
488
489/// 8x8 AVX2+FMA microkernel — 4-way K-unrolled broadcast accumulation.
490/// 8 columns of C in 8 YMM registers, interleaved loads and FMAs for
491/// software pipelining (10-12 instruction distance between load and use).
492/// A: 8×K packed column-major. B: K×8 packed row-major.
493/// C: 8×8 column-major with stride ldc.
494#[cfg(target_arch = "x86_64")]
495#[target_feature(enable = "avx2", enable = "fma")]
496pub unsafe fn microkernel_8x8_avx2_fma(
497    k: usize,
498    a: *const f32,
499    b: *const f32,
500    c: *mut f32,
501    ldc: usize,
502) {
503    unsafe {
504        use std::arch::x86_64::*;
505
506        // Load C (8 columns of 8 elements)
507        let mut c0 = _mm256_loadu_ps(c);
508        let mut c1 = _mm256_loadu_ps(c.add(ldc));
509        let mut c2 = _mm256_loadu_ps(c.add(2 * ldc));
510        let mut c3 = _mm256_loadu_ps(c.add(3 * ldc));
511        let mut c4 = _mm256_loadu_ps(c.add(4 * ldc));
512        let mut c5 = _mm256_loadu_ps(c.add(5 * ldc));
513        let mut c6 = _mm256_loadu_ps(c.add(6 * ldc));
514        let mut c7 = _mm256_loadu_ps(c.add(7 * ldc));
515
516        // 4-way K-unrolled main loop for software pipelining.
517        // Interleaves A loads with B broadcasts and FMAs to hide
518        // 5-cycle FMA latency across 2 FMA ports (Haswell+).
519        let k4 = k / 4;
520        let k_rem = k % 4;
521
522        for p4 in 0..k4 {
523            let base = p4 * 4;
524
525            // K+0: load A, broadcast B, accumulate
526            let a0 = _mm256_loadu_ps(a.add(base * 8));
527            let bp0 = b.add(base * 8);
528            c0 = _mm256_fmadd_ps(a0, _mm256_broadcast_ss(&*bp0), c0);
529            c1 = _mm256_fmadd_ps(a0, _mm256_broadcast_ss(&*bp0.add(1)), c1);
530            c2 = _mm256_fmadd_ps(a0, _mm256_broadcast_ss(&*bp0.add(2)), c2);
531            c3 = _mm256_fmadd_ps(a0, _mm256_broadcast_ss(&*bp0.add(3)), c3);
532            c4 = _mm256_fmadd_ps(a0, _mm256_broadcast_ss(&*bp0.add(4)), c4);
533            c5 = _mm256_fmadd_ps(a0, _mm256_broadcast_ss(&*bp0.add(5)), c5);
534            c6 = _mm256_fmadd_ps(a0, _mm256_broadcast_ss(&*bp0.add(6)), c6);
535            c7 = _mm256_fmadd_ps(a0, _mm256_broadcast_ss(&*bp0.add(7)), c7);
536
537            // K+1
538            let a1 = _mm256_loadu_ps(a.add((base + 1) * 8));
539            let bp1 = b.add((base + 1) * 8);
540            c0 = _mm256_fmadd_ps(a1, _mm256_broadcast_ss(&*bp1), c0);
541            c1 = _mm256_fmadd_ps(a1, _mm256_broadcast_ss(&*bp1.add(1)), c1);
542            c2 = _mm256_fmadd_ps(a1, _mm256_broadcast_ss(&*bp1.add(2)), c2);
543            c3 = _mm256_fmadd_ps(a1, _mm256_broadcast_ss(&*bp1.add(3)), c3);
544            c4 = _mm256_fmadd_ps(a1, _mm256_broadcast_ss(&*bp1.add(4)), c4);
545            c5 = _mm256_fmadd_ps(a1, _mm256_broadcast_ss(&*bp1.add(5)), c5);
546            c6 = _mm256_fmadd_ps(a1, _mm256_broadcast_ss(&*bp1.add(6)), c6);
547            c7 = _mm256_fmadd_ps(a1, _mm256_broadcast_ss(&*bp1.add(7)), c7);
548
549            // K+2
550            let a2 = _mm256_loadu_ps(a.add((base + 2) * 8));
551            let bp2 = b.add((base + 2) * 8);
552            c0 = _mm256_fmadd_ps(a2, _mm256_broadcast_ss(&*bp2), c0);
553            c1 = _mm256_fmadd_ps(a2, _mm256_broadcast_ss(&*bp2.add(1)), c1);
554            c2 = _mm256_fmadd_ps(a2, _mm256_broadcast_ss(&*bp2.add(2)), c2);
555            c3 = _mm256_fmadd_ps(a2, _mm256_broadcast_ss(&*bp2.add(3)), c3);
556            c4 = _mm256_fmadd_ps(a2, _mm256_broadcast_ss(&*bp2.add(4)), c4);
557            c5 = _mm256_fmadd_ps(a2, _mm256_broadcast_ss(&*bp2.add(5)), c5);
558            c6 = _mm256_fmadd_ps(a2, _mm256_broadcast_ss(&*bp2.add(6)), c6);
559            c7 = _mm256_fmadd_ps(a2, _mm256_broadcast_ss(&*bp2.add(7)), c7);
560
561            // K+3
562            let a3 = _mm256_loadu_ps(a.add((base + 3) * 8));
563            let bp3 = b.add((base + 3) * 8);
564            c0 = _mm256_fmadd_ps(a3, _mm256_broadcast_ss(&*bp3), c0);
565            c1 = _mm256_fmadd_ps(a3, _mm256_broadcast_ss(&*bp3.add(1)), c1);
566            c2 = _mm256_fmadd_ps(a3, _mm256_broadcast_ss(&*bp3.add(2)), c2);
567            c3 = _mm256_fmadd_ps(a3, _mm256_broadcast_ss(&*bp3.add(3)), c3);
568            c4 = _mm256_fmadd_ps(a3, _mm256_broadcast_ss(&*bp3.add(4)), c4);
569            c5 = _mm256_fmadd_ps(a3, _mm256_broadcast_ss(&*bp3.add(5)), c5);
570            c6 = _mm256_fmadd_ps(a3, _mm256_broadcast_ss(&*bp3.add(6)), c6);
571            c7 = _mm256_fmadd_ps(a3, _mm256_broadcast_ss(&*bp3.add(7)), c7);
572        }
573
574        // Remainder (k % 4)
575        let base_rem = k4 * 4;
576        for p in 0..k_rem {
577            let pp = base_rem + p;
578            let a_col = _mm256_loadu_ps(a.add(pp * 8));
579            let bp = b.add(pp * 8);
580            c0 = _mm256_fmadd_ps(a_col, _mm256_broadcast_ss(&*bp), c0);
581            c1 = _mm256_fmadd_ps(a_col, _mm256_broadcast_ss(&*bp.add(1)), c1);
582            c2 = _mm256_fmadd_ps(a_col, _mm256_broadcast_ss(&*bp.add(2)), c2);
583            c3 = _mm256_fmadd_ps(a_col, _mm256_broadcast_ss(&*bp.add(3)), c3);
584            c4 = _mm256_fmadd_ps(a_col, _mm256_broadcast_ss(&*bp.add(4)), c4);
585            c5 = _mm256_fmadd_ps(a_col, _mm256_broadcast_ss(&*bp.add(5)), c5);
586            c6 = _mm256_fmadd_ps(a_col, _mm256_broadcast_ss(&*bp.add(6)), c6);
587            c7 = _mm256_fmadd_ps(a_col, _mm256_broadcast_ss(&*bp.add(7)), c7);
588        }
589
590        // Store C
591        _mm256_storeu_ps(c, c0);
592        _mm256_storeu_ps(c.add(ldc), c1);
593        _mm256_storeu_ps(c.add(2 * ldc), c2);
594        _mm256_storeu_ps(c.add(3 * ldc), c3);
595        _mm256_storeu_ps(c.add(4 * ldc), c4);
596        _mm256_storeu_ps(c.add(5 * ldc), c5);
597        _mm256_storeu_ps(c.add(6 * ldc), c6);
598        _mm256_storeu_ps(c.add(7 * ldc), c7);
599    }
600}