#![allow(dead_code)]
#![allow(unsafe_op_in_unsafe_fn)]
#![allow(clippy::too_many_arguments)]
use core::arch::x86_64::*;
#[target_feature(enable = "avx2")]
pub(crate) unsafe fn raw_dot_int(
amp_table_i8: &[i8; 32],
cos_table_i8: &[i8; 64],
sin_table_i8: &[i8; 64],
num_pairs: usize,
amp_stream: &[u8],
phase_stream: &[u8],
q_i8: &[i8],
) -> i64 {
debug_assert!(num_pairs.is_multiple_of(8) && num_pairs >= 8);
let groups = num_pairs / 8;
let safe_groups = groups - 1;
let mut acc0 = _mm256_setzero_si256();
let mut acc1 = _mm256_setzero_si256();
let mut amp_ptr = amp_stream.as_ptr();
let mut phase_ptr = phase_stream.as_ptr();
let mut q_ptr = q_i8.as_ptr();
let q_split_mask = _mm_setr_epi8(
0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9, 11, 13, 15, );
let round_bias = _mm_set1_epi16(1 << 6);
for g in 0..safe_groups {
let prod = process_group(
amp_table_i8,
cos_table_i8,
sin_table_i8,
amp_ptr,
phase_ptr,
q_ptr,
q_split_mask,
round_bias,
);
if g & 1 == 0 {
acc0 = _mm256_add_epi32(acc0, prod);
} else {
acc1 = _mm256_add_epi32(acc1, prod);
}
amp_ptr = amp_ptr.add(5);
phase_ptr = phase_ptr.add(6);
q_ptr = q_ptr.add(16);
}
let mut a_tail = [0u8; 8];
let mut p_tail = [0u8; 8];
core::ptr::copy_nonoverlapping(amp_ptr, a_tail.as_mut_ptr(), 5);
core::ptr::copy_nonoverlapping(phase_ptr, p_tail.as_mut_ptr(), 6);
let prod = process_group_from_chunks(
amp_table_i8,
cos_table_i8,
sin_table_i8,
u64::from_le_bytes(a_tail),
u64::from_le_bytes(p_tail),
q_ptr,
q_split_mask,
round_bias,
);
acc0 = _mm256_add_epi32(acc0, prod);
let acc = _mm256_add_epi32(acc0, acc1);
let lo = _mm256_castsi256_si128(acc);
let hi = _mm256_extracti128_si256(acc, 1);
let s4 = _mm_add_epi32(lo, hi);
let s2 = _mm_add_epi32(s4, _mm_shuffle_epi32(s4, 0b00_00_11_10));
let s1 = _mm_add_epi32(s2, _mm_shuffle_epi32(s2, 0b00_00_00_01));
_mm_cvtsi128_si32(s1) as i64
}
#[target_feature(enable = "avx2")]
#[inline]
unsafe fn process_group(
amp_table_i8: &[i8; 32],
cos_table_i8: &[i8; 64],
sin_table_i8: &[i8; 64],
amp_ptr: *const u8,
phase_ptr: *const u8,
q_ptr: *const i8,
q_split_mask: __m128i,
round_bias: __m128i,
) -> __m256i {
let a_chunk = core::ptr::read_unaligned(amp_ptr as *const u64);
let p_chunk = core::ptr::read_unaligned(phase_ptr as *const u64);
process_group_from_chunks(
amp_table_i8,
cos_table_i8,
sin_table_i8,
a_chunk,
p_chunk,
q_ptr,
q_split_mask,
round_bias,
)
}
#[target_feature(enable = "avx2")]
#[inline]
unsafe fn process_group_from_chunks(
amp_table_i8: &[i8; 32],
cos_table_i8: &[i8; 64],
sin_table_i8: &[i8; 64],
a_chunk: u64,
p_chunk: u64,
q_ptr: *const i8,
q_split_mask: __m128i,
round_bias: __m128i,
) -> __m256i {
let ai_arr: [u8; 8] = [
((a_chunk) & 0x1F) as u8,
((a_chunk >> 5) & 0x1F) as u8,
((a_chunk >> 10) & 0x1F) as u8,
((a_chunk >> 15) & 0x1F) as u8,
((a_chunk >> 20) & 0x1F) as u8,
((a_chunk >> 25) & 0x1F) as u8,
((a_chunk >> 30) & 0x1F) as u8,
((a_chunk >> 35) & 0x1F) as u8,
];
let pi_arr: [u8; 8] = [
((p_chunk) & 0x3F) as u8,
((p_chunk >> 6) & 0x3F) as u8,
((p_chunk >> 12) & 0x3F) as u8,
((p_chunk >> 18) & 0x3F) as u8,
((p_chunk >> 24) & 0x3F) as u8,
((p_chunk >> 30) & 0x3F) as u8,
((p_chunk >> 36) & 0x3F) as u8,
((p_chunk >> 42) & 0x3F) as u8,
];
let mut a_vals = [0i8; 8];
let mut c_vals = [0i8; 8];
let mut s_vals = [0i8; 8];
for i in 0..8 {
a_vals[i] = *amp_table_i8.get_unchecked(ai_arr[i] as usize);
c_vals[i] = *cos_table_i8.get_unchecked(pi_arr[i] as usize);
s_vals[i] = *sin_table_i8.get_unchecked(pi_arr[i] as usize);
}
let q16 = _mm_loadu_si128(q_ptr as *const __m128i);
let q_split = _mm_shuffle_epi8(q16, q_split_mask);
let q_re_i16 = _mm_cvtepi8_epi16(q_split);
let q_im_i16 = _mm_cvtepi8_epi16(_mm_srli_si128(q_split, 8));
let c_i8 = _mm_loadl_epi64(c_vals.as_ptr() as *const __m128i);
let s_i8 = _mm_loadl_epi64(s_vals.as_ptr() as *const __m128i);
let c_i16 = _mm_cvtepi8_epi16(c_i8);
let s_i16 = _mm_cvtepi8_epi16(s_i8);
let q_pairs_lo = _mm_unpacklo_epi16(q_re_i16, q_im_i16);
let q_pairs_hi = _mm_unpackhi_epi16(q_re_i16, q_im_i16);
let cs_pairs_lo = _mm_unpacklo_epi16(c_i16, s_i16);
let cs_pairs_hi = _mm_unpackhi_epi16(c_i16, s_i16);
let inner_lo = _mm_madd_epi16(q_pairs_lo, cs_pairs_lo);
let inner_hi = _mm_madd_epi16(q_pairs_hi, cs_pairs_hi);
let qcs16 = _mm_packs_epi32(inner_lo, inner_hi);
debug_assert_bounds_qcs16(qcs16);
let biased = _mm_add_epi16(qcs16, round_bias);
let shifted = _mm_srai_epi16(biased, 7);
let ip_i8 = _mm_packs_epi16(shifted, shifted);
let a_i8 = _mm_loadl_epi64(a_vals.as_ptr() as *const __m128i);
let ip_i32 = _mm256_cvtepi8_epi32(ip_i8);
let a_i32 = _mm256_cvtepi8_epi32(a_i8);
_mm256_mullo_epi32(ip_i32, a_i32)
}
#[cfg(debug_assertions)]
#[target_feature(enable = "avx2")]
#[inline]
unsafe fn debug_assert_bounds_qcs16(qcs16: __m128i) {
let mut lanes = [0i16; 8];
_mm_storeu_si128(lanes.as_mut_ptr() as *mut __m128i, qcs16);
for (i, &v) in lanes.iter().enumerate() {
debug_assert!(
v.unsigned_abs() <= 16002,
"raw_dot_int_x86_64: |qcs16[{i}]| = {} exceeds the documented 16002 bound \
(q_re∈[-127,127], c,s∈[-63,63]); the round-narrow path's overflow guarantees \
no longer hold",
v.unsigned_abs()
);
}
}
#[cfg(not(debug_assertions))]
#[target_feature(enable = "avx2")]
#[inline]
unsafe fn debug_assert_bounds_qcs16(_qcs16: __m128i) {}