NeuralAmpModeler-rs 3.0.2

// SPDX-License-Identifier: Apache-2.0
// Copyright (c) 2026 Fábio Henrique de Lima Silva (fhl.bsb@gmail.com) All rights reserved.

use super::*;
use crate::math::common::half::f32_to_f16_bits;
use crate::math::common::scalar_ref;

/// Generates test data: weights `[[u16; 4]]` and state `[f32]`.
fn make_data(len: usize) -> (Vec<[u16; 4]>, Vec<f32>) {
    let weights: Vec<[u16; 4]> = (0..len)
        .map(|i| {
            let v = (i as f32 * 0.1).sin() * 0.5 + 0.5;
            let bits = f32_to_f16_bits(v);
            [
                bits,
                bits.wrapping_mul(3),
                bits.wrapping_add(0x1000),
                bits.wrapping_mul(7),
            ]
        })
        .collect();
    let state: Vec<f32> = (0..len).map(|i| (i as f32 * 0.07).sin()).collect();
    (weights, state)
}

#[test]
fn test_dot_4x_interleaved_avx512_vs_fallback() {
    if !std::is_x86_feature_detected!("avx512f") {
        return;
    }

    let sizes = [
        0, 1, 2, 3, 4, 5, 7, 8, 9, 15, 16, 17, 31, 32, 33, 64, 127, 128, 255, 256, 512,
    ];
    for &len in &sizes {
        let (weights, state) = make_data(len);
        let expected = unsafe { scalar_ref::dot_product_4x_interleaved_fallback(&weights, &state) };
        let result = unsafe { dot_product_4x_interleaved_avx512(&weights, &state) };
        for j in 0..4 {
            assert!(
                (result[j] - expected[j]).abs() < 1e-3,
                "len={} channel={}: simd={}, scalar={}",
                len,
                j,
                result[j],
                expected[j]
            );
        }
    }
}

#[test]
fn test_dot_4x_interleaved_dual_frame_avx512_vs_fallback() {
    if !std::is_x86_feature_detected!("avx512f") {
        return;
    }

    let sizes = [
        0, 1, 2, 3, 4, 5, 7, 8, 9, 15, 16, 17, 31, 32, 33, 64, 127, 128, 255, 256, 512,
    ];
    for &len in &sizes {
        let (weights, state_f0) = make_data(len);
        let state_f1: Vec<f32> = state_f0.iter().map(|&s| s * 0.8 + 0.1).collect();

        let expected = unsafe {
            scalar_ref::dot_product_4x_interleaved_dual_frame_fallback(
                &weights, &state_f0, &state_f1,
            )
        };
        let result =
            unsafe { dot_product_4x_interleaved_dual_frame_avx512(&weights, &state_f0, &state_f1) };
        for j in 0..4 {
            assert!(
                (result.0[j] - expected.0[j]).abs() < 1e-3,
                "len={} frame0 channel={}: simd={}, scalar={}",
                len,
                j,
                result.0[j],
                expected.0[j]
            );
            assert!(
                (result.1[j] - expected.1[j]).abs() < 1e-3,
                "len={} frame1 channel={}: simd={}, scalar={}",
                len,
                j,
                result.1[j],
                expected.1[j]
            );
        }
    }
}

#[test]
fn test_dot_4x_interleaved_avx512_stress() {
    if !std::is_x86_feature_detected!("avx512f") {
        return;
    }

    let lengths = [1024, 2048, 4096, 8192];
    for &len in &lengths {
        let (weights, state) = make_data(len);
        let expected = unsafe { scalar_ref::dot_product_4x_interleaved_fallback(&weights, &state) };
        let result = unsafe { dot_product_4x_interleaved_avx512(&weights, &state) };
        for j in 0..4 {
            assert!(
                (result[j] - expected[j]).abs() < 1e-3,
                "stress len={} channel={}: simd={}, scalar={}",
                len,
                j,
                result[j],
                expected[j]
            );
        }
    }
}

#[test]
fn test_dot_4x_interleaved_dual_frame_avx512_stress() {
    if !std::is_x86_feature_detected!("avx512f") {
        return;
    }

    let lengths = [1024, 2048, 4096, 8192];
    for &len in &lengths {
        let (weights, state_f0) = make_data(len);
        let state_f1: Vec<f32> = state_f0.iter().map(|&s| s * 0.8 + 0.1).collect();

        let expected = unsafe {
            scalar_ref::dot_product_4x_interleaved_dual_frame_fallback(
                &weights, &state_f0, &state_f1,
            )
        };
        let result =
            unsafe { dot_product_4x_interleaved_dual_frame_avx512(&weights, &state_f0, &state_f1) };
        for j in 0..4 {
            assert!(
                (result.0[j] - expected.0[j]).abs() < 1e-3,
                "stress len={} f0 ch={}: simd={}, scalar={}",
                len,
                j,
                result.0[j],
                expected.0[j]
            );
            assert!(
                (result.1[j] - expected.1[j]).abs() < 1e-3,
                "stress len={} f1 ch={}: simd={}, scalar={}",
                len,
                j,
                result.1[j],
                expected.1[j]
            );
        }
    }
}

#[test]
fn test_dot_4x_interleaved_avx512_vs_avx2() {
    if !std::is_x86_feature_detected!("avx512f") {
        return;
    }

    let sizes = [0, 1, 3, 8, 16, 64, 128, 256, 512, 1024];
    for &len in &sizes {
        let (weights, state) = make_data(len);
        let avx2_result = unsafe { dot_product_4x_interleaved_avx2(&weights, &state) };
        let avx512_result = unsafe { dot_product_4x_interleaved_avx512(&weights, &state) };
        for j in 0..4 {
            assert!(
                (avx512_result[j] - avx2_result[j]).abs() < 1e-3,
                "len={} channel={}: avx512={}, avx2={}",
                len,
                j,
                avx512_result[j],
                avx2_result[j]
            );
        }
    }
}

#[test]
fn test_dot_4x_interleaved_dual_frame_avx512_vs_avx2() {
    if !std::is_x86_feature_detected!("avx512f") {
        return;
    }

    let sizes = [0, 1, 3, 8, 16, 64, 128, 256, 512, 1024];
    for &len in &sizes {
        let (weights, state_f0) = make_data(len);
        let state_f1: Vec<f32> = state_f0.iter().map(|&s| s * 0.8 + 0.1).collect();

        let avx2_result =
            unsafe { dot_product_4x_interleaved_dual_frame_avx2(&weights, &state_f0, &state_f1) };
        let avx512_result =
            unsafe { dot_product_4x_interleaved_dual_frame_avx512(&weights, &state_f0, &state_f1) };
        for j in 0..4 {
            assert!(
                (avx512_result.0[j] - avx2_result.0[j]).abs() < 1e-3,
                "len={} f0 ch={}: avx512={}, avx2={}",
                len,
                j,
                avx512_result.0[j],
                avx2_result.0[j]
            );
            assert!(
                (avx512_result.1[j] - avx2_result.1[j]).abs() < 1e-3,
                "len={} f1 ch={}: avx512={}, avx2={}",
                len,
                j,
                avx512_result.1[j],
                avx2_result.1[j]
            );
        }
    }
}

// ── F32-native dot_4x tests (f32 weights, no quantization) ──────────────────

fn make_f32_data(len: usize) -> (Vec<[f32; 4]>, Vec<f32>) {
    let weights: Vec<[f32; 4]> = (0..len)
        .map(|i| {
            let v = (i as f32 * 0.1).sin() * 0.5 + 0.5;
            [v, v * 1.5, v * 0.7, v * 2.3]
        })
        .collect();
    let state: Vec<f32> = (0..len).map(|i| (i as f32 * 0.07).sin()).collect();
    (weights, state)
}

#[test]
fn test_dot_4x_f32_avx2_vs_scalar() {
    let sizes = [
        0, 1, 2, 3, 4, 5, 7, 8, 9, 15, 16, 17, 31, 32, 33, 64, 127, 128, 255, 256, 512,
    ];
    for &len in &sizes {
        let (weights, state) = make_f32_data(len);
        let expected = unsafe { scalar_ref::dot_product_4x_f32_scalar(&weights, &state) };
        let result = unsafe { dot_product_4x_f32_avx2(&weights, &state) };
        for j in 0..4 {
            assert!(
                (result[j] - expected[j]).abs() < 5e-4,
                "len={} channel={}: avx2={}, scalar={}",
                len,
                j,
                result[j],
                expected[j]
            );
        }
    }
}

#[test]
fn test_dot_4x_f32_avx2_stress() {
    let lengths = [1024, 2048, 4096, 8192];
    for &len in &lengths {
        let (weights, state) = make_f32_data(len);
        let expected = unsafe { scalar_ref::dot_product_4x_f32_scalar(&weights, &state) };
        let result = unsafe { dot_product_4x_f32_avx2(&weights, &state) };
        for j in 0..4 {
            assert!(
                (result[j] - expected[j]).abs() < 2e-3,
                "stress len={} channel={}: avx2={}, scalar={}",
                len,
                j,
                result[j],
                expected[j]
            );
        }
    }
}

// ── F32-native dot_4x dual-frame tests (f32 weights, no quantization) ─────

#[test]
fn test_dot_4x_f32_dual_avx2_vs_scalar() {
    let sizes = [
        0, 1, 2, 3, 4, 5, 7, 8, 9, 15, 16, 17, 31, 32, 33, 64, 127, 128, 255, 256, 512,
    ];
    for &len in &sizes {
        let (weights, state_f0) = make_f32_data(len);
        let state_f1: Vec<f32> = state_f0.iter().map(|&s| s * 0.8 + 0.1).collect();

        let expected =
            unsafe { scalar_ref::dot_product_4x_f32_dual_scalar(&weights, &state_f0, &state_f1) };
        let result = unsafe { dot_product_4x_f32_dual_avx2(&weights, &state_f0, &state_f1) };
        for j in 0..4 {
            assert!(
                (result.0[j] - expected.0[j]).abs() < 5e-4,
                "len={} f0 ch={}: avx2={}, scalar={}",
                len,
                j,
                result.0[j],
                expected.0[j]
            );
            assert!(
                (result.1[j] - expected.1[j]).abs() < 5e-4,
                "len={} f1 ch={}: avx2={}, scalar={}",
                len,
                j,
                result.1[j],
                expected.1[j]
            );
        }
    }
}

#[test]
fn test_dot_4x_f32_dual_avx2_single_vs_dual_invariance() {
    let sizes = [
        0, 1, 2, 3, 4, 5, 7, 8, 9, 15, 16, 17, 31, 32, 33, 64, 127, 128, 255, 256, 512,
    ];
    for &len in &sizes {
        let (weights, state_f0) = make_f32_data(len);
        let state_f1: Vec<f32> = state_f0.iter().map(|&s| s * 0.8 + 0.1).collect();

        let single_f0 = unsafe { dot_product_4x_f32_avx2(&weights, &state_f0) };
        let single_f1 = unsafe { dot_product_4x_f32_avx2(&weights, &state_f1) };
        let dual = unsafe { dot_product_4x_f32_dual_avx2(&weights, &state_f0, &state_f1) };
        for j in 0..4 {
            assert!(
                (dual.0[j] - single_f0[j]).abs() < 5e-4,
                "len={} f0 ch={}: dual={}, single={}",
                len,
                j,
                dual.0[j],
                single_f0[j]
            );
            assert!(
                (dual.1[j] - single_f1[j]).abs() < 5e-4,
                "len={} f1 ch={}: dual={}, single={}",
                len,
                j,
                dual.1[j],
                single_f1[j]
            );
        }
    }
}

#[test]
fn test_dot_4x_f32_dual_avx2_stress() {
    let lengths = [1024, 2048, 4096, 8192];
    for &len in &lengths {
        let (weights, state_f0) = make_f32_data(len);
        let state_f1: Vec<f32> = state_f0.iter().map(|&s| s * 0.8 + 0.1).collect();

        let expected =
            unsafe { scalar_ref::dot_product_4x_f32_dual_scalar(&weights, &state_f0, &state_f1) };
        let result = unsafe { dot_product_4x_f32_dual_avx2(&weights, &state_f0, &state_f1) };
        for j in 0..4 {
            assert!(
                (result.0[j] - expected.0[j]).abs() < 5e-3,
                "stress len={} f0 ch={}: avx2={}, scalar={}",
                len,
                j,
                result.0[j],
                expected.0[j]
            );
            assert!(
                (result.1[j] - expected.1[j]).abs() < 5e-3,
                "stress len={} f1 ch={}: avx2={}, scalar={}",
                len,
                j,
                result.1[j],
                expected.1[j]
            );
        }
    }
}

// ── F32-native dot_4x AVX-512 tests (f32 weights, no quantization) ─────────

#[test]
fn test_dot_4x_f32_avx512_vs_scalar() {
    if !std::is_x86_feature_detected!("avx512f") {
        return;
    }

    let sizes = [
        0, 1, 2, 3, 4, 5, 7, 8, 9, 15, 16, 17, 31, 32, 33, 64, 127, 128, 255, 256, 512,
    ];
    for &len in &sizes {
        let (weights, state) = make_f32_data(len);
        let expected = unsafe { scalar_ref::dot_product_4x_f32_scalar(&weights, &state) };
        let result = unsafe { dot_product_4x_f32_avx512(&weights, &state) };
        for j in 0..4 {
            assert_eq!(
                result[j], expected[j],
                "len={} channel={}: avx512={}, scalar={}",
                len, j, result[j], expected[j]
            );
        }
    }
}

#[test]
fn test_dot_4x_f32_avx512_vs_avx2() {
    if !std::is_x86_feature_detected!("avx512f") {
        return;
    }

    let sizes = [
        0, 1, 2, 3, 4, 5, 7, 8, 9, 15, 16, 17, 31, 32, 33, 64, 127, 128, 255, 256, 512,
    ];
    for &len in &sizes {
        let (weights, state) = make_f32_data(len);
        let avx2_result = unsafe { dot_product_4x_f32_avx2(&weights, &state) };
        let avx512_result = unsafe { dot_product_4x_f32_avx512(&weights, &state) };
        for j in 0..4 {
            assert!(
                (avx512_result[j] - avx2_result[j]).abs() < 5e-4,
                "len={} channel={}: avx512={}, avx2={}",
                len,
                j,
                avx512_result[j],
                avx2_result[j]
            );
        }
    }
}

#[test]
fn test_dot_4x_f32_avx512_stress() {
    if !std::is_x86_feature_detected!("avx512f") {
        return;
    }

    let lengths = [1024, 2048, 4096, 8192];
    for &len in &lengths {
        let (weights, state) = make_f32_data(len);
        let expected = unsafe { scalar_ref::dot_product_4x_f32_scalar(&weights, &state) };
        let result = unsafe { dot_product_4x_f32_avx512(&weights, &state) };
        for j in 0..4 {
            assert_eq!(
                result[j], expected[j],
                "stress len={} channel={}: avx512={}, scalar={}",
                len, j, result[j], expected[j]
            );
        }
    }
}

#[test]
fn test_dot_4x_f32_dual_avx512_vs_scalar() {
    if !std::is_x86_feature_detected!("avx512f") {
        return;
    }

    let sizes = [
        0, 1, 2, 3, 4, 5, 7, 8, 9, 15, 16, 17, 31, 32, 33, 64, 127, 128, 255, 256, 512,
    ];
    for &len in &sizes {
        let (weights, state_f0) = make_f32_data(len);
        let state_f1: Vec<f32> = state_f0.iter().map(|&s| s * 0.8 + 0.1).collect();

        let expected =
            unsafe { scalar_ref::dot_product_4x_f32_dual_scalar(&weights, &state_f0, &state_f1) };
        let result = unsafe { dot_product_4x_f32_dual_avx512(&weights, &state_f0, &state_f1) };
        for j in 0..4 {
            assert_eq!(
                result.0[j], expected.0[j],
                "len={} f0 ch={}: avx512={}, scalar={}",
                len, j, result.0[j], expected.0[j]
            );
            assert_eq!(
                result.1[j], expected.1[j],
                "len={} f1 ch={}: avx512={}, scalar={}",
                len, j, result.1[j], expected.1[j]
            );
        }
    }
}

#[test]
fn test_dot_4x_f32_dual_avx512_vs_avx2() {
    if !std::is_x86_feature_detected!("avx512f") {
        return;
    }

    let sizes = [
        0, 1, 2, 3, 4, 5, 7, 8, 9, 15, 16, 17, 31, 32, 33, 64, 127, 128, 255, 256, 512,
    ];
    for &len in &sizes {
        let (weights, state_f0) = make_f32_data(len);
        let state_f1: Vec<f32> = state_f0.iter().map(|&s| s * 0.8 + 0.1).collect();

        let avx2_result = unsafe { dot_product_4x_f32_dual_avx2(&weights, &state_f0, &state_f1) };
        let avx512_result =
            unsafe { dot_product_4x_f32_dual_avx512(&weights, &state_f0, &state_f1) };
        for j in 0..4 {
            assert!(
                (avx512_result.0[j] - avx2_result.0[j]).abs() < 5e-4,
                "len={} f0 ch={}: avx512={}, avx2={}",
                len,
                j,
                avx512_result.0[j],
                avx2_result.0[j]
            );
            assert!(
                (avx512_result.1[j] - avx2_result.1[j]).abs() < 5e-4,
                "len={} f1 ch={}: avx512={}, avx2={}",
                len,
                j,
                avx512_result.1[j],
                avx2_result.1[j]
            );
        }
    }
}

#[test]
fn test_dot_4x_f32_dual_avx512_single_vs_dual_invariance() {
    if !std::is_x86_feature_detected!("avx512f") {
        return;
    }

    let sizes = [
        0, 1, 2, 3, 4, 5, 7, 8, 9, 15, 16, 17, 31, 32, 33, 64, 127, 128, 255, 256, 512,
    ];
    for &len in &sizes {
        let (weights, state_f0) = make_f32_data(len);
        let state_f1: Vec<f32> = state_f0.iter().map(|&s| s * 0.8 + 0.1).collect();

        let single_f0 = unsafe { dot_product_4x_f32_avx512(&weights, &state_f0) };
        let single_f1 = unsafe { dot_product_4x_f32_avx512(&weights, &state_f1) };
        let dual = unsafe { dot_product_4x_f32_dual_avx512(&weights, &state_f0, &state_f1) };
        for j in 0..4 {
            assert_eq!(
                dual.0[j], single_f0[j],
                "len={} f0 ch={}: dual={}, single={}",
                len, j, dual.0[j], single_f0[j]
            );
            assert_eq!(
                dual.1[j], single_f1[j],
                "len={} f1 ch={}: dual={}, single={}",
                len, j, dual.1[j], single_f1[j]
            );
        }
    }
}

#[test]
fn test_dot_4x_f32_dual_avx512_stress() {
    if !std::is_x86_feature_detected!("avx512f") {
        return;
    }

    let lengths = [1024, 2048, 4096, 8192];
    for &len in &lengths {
        let (weights, state_f0) = make_f32_data(len);
        let state_f1: Vec<f32> = state_f0.iter().map(|&s| s * 0.8 + 0.1).collect();

        let expected =
            unsafe { scalar_ref::dot_product_4x_f32_dual_scalar(&weights, &state_f0, &state_f1) };
        let result = unsafe { dot_product_4x_f32_dual_avx512(&weights, &state_f0, &state_f1) };
        for j in 0..4 {
            assert_eq!(
                result.0[j], expected.0[j],
                "stress len={} f0 ch={}: avx512={}, scalar={}",
                len, j, result.0[j], expected.0[j]
            );
            assert_eq!(
                result.1[j], expected.1[j],
                "stress len={} f1 ch={}: avx512={}, scalar={}",
                len, j, result.1[j], expected.1[j]
            );
        }
    }
}

// ── Accumulate kernel tests (f32 weights + init) ─────────────────────────────

fn make_f32_init_4() -> [f32; 4] {
    [0.1, -0.25, 0.05, 0.4]
}

#[test]
fn test_dot_4x_f32_accumulate_avx2_vs_scalar() {
    let sizes = [
        0, 1, 2, 3, 4, 5, 7, 8, 9, 15, 16, 17, 31, 32, 33, 64, 127, 128, 255, 256, 512,
    ];
    let init = make_f32_init_4();
    for &len in &sizes {
        let (weights, state) = make_f32_data(len);
        let expected =
            unsafe { scalar_ref::dot_product_4x_f32_accumulate_scalar(&weights, &state, &init) };
        let result = unsafe { dot_product_4x_f32_accumulate_avx2(&weights, &state, &init) };
        for j in 0..4 {
            assert!(
                (result[j] - expected[j]).abs() < 5e-4,
                "len={} channel={}: avx2={}, scalar={}",
                len,
                j,
                result[j],
                expected[j]
            );
        }
    }
}

#[test]
fn test_dot_4x_f32_accumulate_avx2_stress() {
    let lengths = [1024, 2048, 4096, 8192];
    let init = make_f32_init_4();
    for &len in &lengths {
        let (weights, state) = make_f32_data(len);
        let expected =
            unsafe { scalar_ref::dot_product_4x_f32_accumulate_scalar(&weights, &state, &init) };
        let result = unsafe { dot_product_4x_f32_accumulate_avx2(&weights, &state, &init) };
        for j in 0..4 {
            assert!(
                (result[j] - expected[j]).abs() < 2e-3,
                "stress len={} channel={}: avx2={}, scalar={}",
                len,
                j,
                result[j],
                expected[j]
            );
        }
    }
}

fn make_f32_init_dual_4() -> ([f32; 4], [f32; 4]) {
    ([0.1, -0.25, 0.05, 0.4], [-0.15, 0.3, -0.05, 0.2])
}

#[test]
fn test_dot_4x_f32_dual_accumulate_avx2_vs_scalar() {
    let sizes = [
        0, 1, 2, 3, 4, 5, 7, 8, 9, 15, 16, 17, 31, 32, 33, 64, 127, 128, 255, 256, 512,
    ];
    let (init_f0, init_f1) = make_f32_init_dual_4();
    for &len in &sizes {
        let (weights, state_f0) = make_f32_data(len);
        let state_f1: Vec<f32> = state_f0.iter().map(|&s| s * 0.8 + 0.1).collect();

        let expected = unsafe {
            scalar_ref::dot_product_4x_f32_dual_accumulate_scalar(
                &weights, &state_f0, &state_f1, &init_f0, &init_f1,
            )
        };
        let result = unsafe {
            dot_product_4x_f32_dual_accumulate_avx2(
                &weights, &state_f0, &state_f1, &init_f0, &init_f1,
            )
        };
        for j in 0..4 {
            assert!(
                (result.0[j] - expected.0[j]).abs() < 5e-4,
                "len={} f0 ch={}: avx2={}, scalar={}",
                len,
                j,
                result.0[j],
                expected.0[j]
            );
            assert!(
                (result.1[j] - expected.1[j]).abs() < 5e-4,
                "len={} f1 ch={}: avx2={}, scalar={}",
                len,
                j,
                result.1[j],
                expected.1[j]
            );
        }
    }
}

#[test]
fn test_dot_4x_f32_dual_accumulate_avx2_stress() {
    let lengths = [1024, 2048, 4096, 8192];
    let (init_f0, init_f1) = make_f32_init_dual_4();
    for &len in &lengths {
        let (weights, state_f0) = make_f32_data(len);
        let state_f1: Vec<f32> = state_f0.iter().map(|&s| s * 0.8 + 0.1).collect();

        let expected = unsafe {
            scalar_ref::dot_product_4x_f32_dual_accumulate_scalar(
                &weights, &state_f0, &state_f1, &init_f0, &init_f1,
            )
        };
        let result = unsafe {
            dot_product_4x_f32_dual_accumulate_avx2(
                &weights, &state_f0, &state_f1, &init_f0, &init_f1,
            )
        };
        for j in 0..4 {
            assert!(
                (result.0[j] - expected.0[j]).abs() < 5e-3,
                "stress len={} f0 ch={}: avx2={}, scalar={}",
                len,
                j,
                result.0[j],
                expected.0[j]
            );
            assert!(
                (result.1[j] - expected.1[j]).abs() < 5e-3,
                "stress len={} f1 ch={}: avx2={}, scalar={}",
                len,
                j,
                result.1[j],
                expected.1[j]
            );
        }
    }
}

// ── AVX-512 accumulate kernel tests (f32) ─────────────────────────────────

#[test]
fn test_dot_4x_f32_accumulate_avx512_vs_scalar() {
    if !std::is_x86_feature_detected!("avx512f") {
        return;
    }

    let sizes = [
        0, 1, 2, 3, 4, 5, 7, 8, 9, 15, 16, 17, 31, 32, 33, 64, 127, 128, 255, 256, 512,
    ];
    let init = make_f32_init_4();
    for &len in &sizes {
        let (weights, state) = make_f32_data(len);
        let expected =
            unsafe { scalar_ref::dot_product_4x_f32_accumulate_scalar(&weights, &state, &init) };
        let result = unsafe { dot_product_4x_f32_accumulate_avx512(&weights, &state, &init) };
        for j in 0..4 {
            assert!(
                (result[j] - expected[j]).abs() < 1e-5,
                "len={} channel={}: avx512={}, scalar={}",
                len,
                j,
                result[j],
                expected[j]
            );
        }
    }
}

#[test]
fn test_dot_4x_f32_accumulate_avx512_stress() {
    if !std::is_x86_feature_detected!("avx512f") {
        return;
    }

    let lengths = [1024, 2048, 4096, 8192];
    let init = make_f32_init_4();
    for &len in &lengths {
        let (weights, state) = make_f32_data(len);
        let expected =
            unsafe { scalar_ref::dot_product_4x_f32_accumulate_scalar(&weights, &state, &init) };
        let result = unsafe { dot_product_4x_f32_accumulate_avx512(&weights, &state, &init) };
        for j in 0..4 {
            assert!(
                (result[j] - expected[j]).abs() < 1e-5,
                "stress len={} channel={}: avx512={}, scalar={}",
                len,
                j,
                result[j],
                expected[j]
            );
        }
    }
}

#[test]
fn test_dot_4x_f32_dual_accumulate_avx512_vs_scalar() {
    if !std::is_x86_feature_detected!("avx512f") {
        return;
    }

    let sizes = [
        0, 1, 2, 3, 4, 5, 7, 8, 9, 15, 16, 17, 31, 32, 33, 64, 127, 128, 255, 256, 512,
    ];
    let (init_f0, init_f1) = make_f32_init_dual_4();
    for &len in &sizes {
        let (weights, state_f0) = make_f32_data(len);
        let state_f1: Vec<f32> = state_f0.iter().map(|&s| s * 0.8 + 0.1).collect();

        let expected = unsafe {
            scalar_ref::dot_product_4x_f32_dual_accumulate_scalar(
                &weights, &state_f0, &state_f1, &init_f0, &init_f1,
            )
        };
        let result = unsafe {
            dot_product_4x_f32_dual_accumulate_avx512(
                &weights, &state_f0, &state_f1, &init_f0, &init_f1,
            )
        };
        for j in 0..4 {
            assert!(
                (result.0[j] - expected.0[j]).abs() < 1e-5,
                "len={} f0 ch={}: avx512={}, scalar={}",
                len,
                j,
                result.0[j],
                expected.0[j]
            );
            assert!(
                (result.1[j] - expected.1[j]).abs() < 1e-5,
                "len={} f1 ch={}: avx512={}, scalar={}",
                len,
                j,
                result.1[j],
                expected.1[j]
            );
        }
    }
}

#[test]
fn test_dot_4x_f32_dual_accumulate_avx512_stress() {
    if !std::is_x86_feature_detected!("avx512f") {
        return;
    }

    let lengths = [1024, 2048, 4096, 8192];
    let (init_f0, init_f1) = make_f32_init_dual_4();
    for &len in &lengths {
        let (weights, state_f0) = make_f32_data(len);
        let state_f1: Vec<f32> = state_f0.iter().map(|&s| s * 0.8 + 0.1).collect();

        let expected = unsafe {
            scalar_ref::dot_product_4x_f32_dual_accumulate_scalar(
                &weights, &state_f0, &state_f1, &init_f0, &init_f1,
            )
        };
        let result = unsafe {
            dot_product_4x_f32_dual_accumulate_avx512(
                &weights, &state_f0, &state_f1, &init_f0, &init_f1,
            )
        };
        for j in 0..4 {
            assert!(
                (result.0[j] - expected.0[j]).abs() < 1e-5,
                "stress len={} f0 ch={}: avx512={}, scalar={}",
                len,
                j,
                result.0[j],
                expected.0[j]
            );
            assert!(
                (result.1[j] - expected.1[j]).abs() < 1e-5,
                "stress len={} f1 ch={}: avx512={}, scalar={}",
                len,
                j,
                result.1[j],
                expected.1[j]
            );
        }
    }
}