#![allow(unsafe_op_in_unsafe_fn)]
use std::arch::aarch64::*;
use std::arch::asm;
#[inline]
#[target_feature(enable = "dotprod")]
unsafe fn sdot_8(acc: int32x2_t, a: int8x8_t, b: int8x8_t) -> int32x2_t {
let mut out: int32x2_t = acc;
asm!(
"sdot {0:v}.2s, {1:v}.8b, {2:v}.8b",
inout(vreg) out,
in(vreg) a,
in(vreg) b,
options(pure, nomem, nostack),
);
out
}
#[target_feature(enable = "neon,dotprod")]
pub(in crate::codec::qam_sliding) unsafe fn raw_dot_int(
amp_table_i8: &[i8; 32],
cos_table_i8: &[i8; 64],
sin_table_i8: &[i8; 64],
num_pairs: usize,
amp_stream: &[u8],
phase_stream: &[u8],
q_i8: &[i8],
) -> i64 {
let a_tbl: int8x16x2_t = vld1q_s8_x2(amp_table_i8.as_ptr());
let c_tbl: int8x16x4_t = vld1q_s8_x4(cos_table_i8.as_ptr());
let s_tbl: int8x16x4_t = vld1q_s8_x4(sin_table_i8.as_ptr());
let groups = num_pairs / 8;
let safe_groups = groups - 1;
let mut acc0 = vdup_n_s32(0);
let mut acc1 = vdup_n_s32(0);
let mut amp_ptr = amp_stream.as_ptr();
let mut phase_ptr = phase_stream.as_ptr();
let mut q_ptr = q_i8.as_ptr();
for g in 0..safe_groups {
let a_chunk = std::ptr::read_unaligned(amp_ptr as *const u64);
let p_chunk = std::ptr::read_unaligned(phase_ptr as *const u64);
let ai_arr: [u8; 8] = [
((a_chunk) & 0x1F) as u8,
((a_chunk >> 5) & 0x1F) as u8,
((a_chunk >> 10) & 0x1F) as u8,
((a_chunk >> 15) & 0x1F) as u8,
((a_chunk >> 20) & 0x1F) as u8,
((a_chunk >> 25) & 0x1F) as u8,
((a_chunk >> 30) & 0x1F) as u8,
((a_chunk >> 35) & 0x1F) as u8,
];
let pi_arr: [u8; 8] = [
((p_chunk) & 0x3F) as u8,
((p_chunk >> 6) & 0x3F) as u8,
((p_chunk >> 12) & 0x3F) as u8,
((p_chunk >> 18) & 0x3F) as u8,
((p_chunk >> 24) & 0x3F) as u8,
((p_chunk >> 30) & 0x3F) as u8,
((p_chunk >> 36) & 0x3F) as u8,
((p_chunk >> 42) & 0x3F) as u8,
];
let ai_vec = vld1_u8(ai_arr.as_ptr());
let pi_vec = vld1_u8(pi_arr.as_ptr());
let a_val = vqtbl2_s8(a_tbl, ai_vec);
let c_val = vqtbl4_s8(c_tbl, pi_vec);
let s_val = vqtbl4_s8(s_tbl, pi_vec);
let q_pair = vld2_s8(q_ptr);
let q_re = q_pair.0;
let q_im = q_pair.1;
let qc16 = vmull_s8(q_re, c_val);
let qcs16 = vmlal_s8(qc16, q_im, s_val);
let ip = vqshrn_n_s16(qcs16, 7);
if g & 1 == 0 {
acc0 = sdot_8(acc0, ip, a_val);
} else {
acc1 = sdot_8(acc1, ip, a_val);
}
amp_ptr = amp_ptr.add(5);
phase_ptr = phase_ptr.add(6);
q_ptr = q_ptr.add(16);
}
let mut a_tail = [0u8; 8];
let mut p_tail = [0u8; 8];
std::ptr::copy_nonoverlapping(amp_ptr, a_tail.as_mut_ptr(), 5);
std::ptr::copy_nonoverlapping(phase_ptr, p_tail.as_mut_ptr(), 6);
let a_chunk = u64::from_le_bytes(a_tail);
let p_chunk = u64::from_le_bytes(p_tail);
let ai_arr: [u8; 8] = [
((a_chunk) & 0x1F) as u8,
((a_chunk >> 5) & 0x1F) as u8,
((a_chunk >> 10) & 0x1F) as u8,
((a_chunk >> 15) & 0x1F) as u8,
((a_chunk >> 20) & 0x1F) as u8,
((a_chunk >> 25) & 0x1F) as u8,
((a_chunk >> 30) & 0x1F) as u8,
((a_chunk >> 35) & 0x1F) as u8,
];
let pi_arr: [u8; 8] = [
((p_chunk) & 0x3F) as u8,
((p_chunk >> 6) & 0x3F) as u8,
((p_chunk >> 12) & 0x3F) as u8,
((p_chunk >> 18) & 0x3F) as u8,
((p_chunk >> 24) & 0x3F) as u8,
((p_chunk >> 30) & 0x3F) as u8,
((p_chunk >> 36) & 0x3F) as u8,
((p_chunk >> 42) & 0x3F) as u8,
];
let ai_vec = vld1_u8(ai_arr.as_ptr());
let pi_vec = vld1_u8(pi_arr.as_ptr());
let a_val = vqtbl2_s8(a_tbl, ai_vec);
let c_val = vqtbl4_s8(c_tbl, pi_vec);
let s_val = vqtbl4_s8(s_tbl, pi_vec);
let q_pair = vld2_s8(q_ptr);
let qc16 = vmull_s8(q_pair.0, c_val);
let qcs16 = vmlal_s8(qc16, q_pair.1, s_val);
let ip = vqshrn_n_s16(qcs16, 7);
acc0 = sdot_8(acc0, ip, a_val);
let acc = vadd_s32(acc0, acc1);
let lane0 = vget_lane_s32(acc, 0) as i64;
let lane1 = vget_lane_s32(acc, 1) as i64;
lane0 + lane1
}