use crate::avx::ac_strategy::hsum256;
use crate::dct::{DctFn, DctInput};
use crate::structure_aq::{Features, finish_block_features};
use std::arch::x86_64::*;
#[inline]
#[target_feature(enable = "avx2,fma")]
fn first_lanes_mask(n: usize) -> __m256i {
_mm256_cmpgt_epi32(
_mm256_set1_epi32(n as i32),
_mm256_setr_epi32(0, 1, 2, 3, 4, 5, 6, 7),
)
}
#[inline(never)]
#[target_feature(enable = "avx2,fma")]
fn moments(block: &[f32; 64]) -> (f32, f32) {
let chunks = block.as_chunks::<8>().0;
let mut sum0 = _mm256_setzero_ps();
let mut sum1 = _mm256_setzero_ps();
let mut sum2 = _mm256_setzero_ps();
let mut sum3 = _mm256_setzero_ps();
for group in 0..2 {
let chunk = group * 4;
sum0 = _mm256_add_ps(sum0, unsafe { _mm256_loadu_ps(chunks[chunk].as_ptr()) });
sum1 = _mm256_add_ps(sum1, unsafe { _mm256_loadu_ps(chunks[chunk + 1].as_ptr()) });
sum2 = _mm256_add_ps(sum2, unsafe { _mm256_loadu_ps(chunks[chunk + 2].as_ptr()) });
sum3 = _mm256_add_ps(sum3, unsafe { _mm256_loadu_ps(chunks[chunk + 3].as_ptr()) });
}
let sum = _mm256_add_ps(_mm256_add_ps(sum0, sum1), _mm256_add_ps(sum2, sum3));
let mean = hsum256(sum) * (1.0 / 64.0);
let mean_v = _mm256_set1_ps(mean);
let mut variance0 = _mm256_setzero_ps();
let mut variance1 = _mm256_setzero_ps();
let mut variance2 = _mm256_setzero_ps();
let mut variance3 = _mm256_setzero_ps();
for group in 0..2 {
let chunk = group * 4;
let d0 = _mm256_sub_ps(unsafe { _mm256_loadu_ps(chunks[chunk].as_ptr()) }, mean_v);
let d1 = _mm256_sub_ps(
unsafe { _mm256_loadu_ps(chunks[chunk + 1].as_ptr()) },
mean_v,
);
let d2 = _mm256_sub_ps(
unsafe { _mm256_loadu_ps(chunks[chunk + 2].as_ptr()) },
mean_v,
);
let d3 = _mm256_sub_ps(
unsafe { _mm256_loadu_ps(chunks[chunk + 3].as_ptr()) },
mean_v,
);
variance0 = _mm256_fmadd_ps(d0, d0, variance0);
variance1 = _mm256_fmadd_ps(d1, d1, variance1);
variance2 = _mm256_fmadd_ps(d2, d2, variance2);
variance3 = _mm256_fmadd_ps(d3, d3, variance3);
}
let variance = _mm256_add_ps(
_mm256_add_ps(variance0, variance1),
_mm256_add_ps(variance2, variance3),
);
(mean, hsum256(variance) * (1.0 / 64.0))
}
#[inline]
#[target_feature(enable = "avx2,fma")]
fn tensor_gradients(rows: &[[f32; 8]], y: usize, mask: __m256i) -> (__m256, __m256) {
let left = unsafe { _mm256_maskload_ps(rows[y].as_ptr(), mask) };
let right = unsafe { _mm256_maskload_ps(rows[y].as_ptr().add(2), mask) };
let top = unsafe { _mm256_maskload_ps(rows[y - 1].as_ptr().add(1), mask) };
let bottom = unsafe { _mm256_maskload_ps(rows[y + 1].as_ptr().add(1), mask) };
(
_mm256_mul_ps(_mm256_sub_ps(right, left), _mm256_set1_ps(0.5)),
_mm256_mul_ps(_mm256_sub_ps(bottom, top), _mm256_set1_ps(0.5)),
)
}
#[inline(never)]
#[target_feature(enable = "avx2,fma")]
fn tensor(block: &[f32; 64]) -> [f32; 3] {
let rows = block.as_chunks::<8>().0;
let mask = first_lanes_mask(6);
let mut jxx0 = _mm256_setzero_ps();
let mut jxy0 = _mm256_setzero_ps();
let mut jyy0 = _mm256_setzero_ps();
let mut jxx1 = _mm256_setzero_ps();
let mut jxy1 = _mm256_setzero_ps();
let mut jyy1 = _mm256_setzero_ps();
for y in (1..7).step_by(2) {
let (gx0, gy0) = tensor_gradients(rows, y, mask);
jxx0 = _mm256_fmadd_ps(gx0, gx0, jxx0);
jxy0 = _mm256_fmadd_ps(gx0, gy0, jxy0);
jyy0 = _mm256_fmadd_ps(gy0, gy0, jyy0);
let (gx1, gy1) = tensor_gradients(rows, y + 1, mask);
jxx1 = _mm256_fmadd_ps(gx1, gx1, jxx1);
jxy1 = _mm256_fmadd_ps(gx1, gy1, jxy1);
jyy1 = _mm256_fmadd_ps(gy1, gy1, jyy1);
}
[
hsum256(_mm256_add_ps(jxx0, jxx1)),
hsum256(_mm256_add_ps(jxy0, jxy1)),
hsum256(_mm256_add_ps(jyy0, jyy1)),
]
}
#[inline(never)]
#[target_feature(enable = "avx2,fma")]
fn predictor_errors(block: &[f32; 64]) -> [f32; 5] {
let rows = block.as_chunks::<8>().0;
let mask = first_lanes_mask(6);
let mut error0 = _mm256_setzero_ps();
let mut error1 = _mm256_setzero_ps();
let mut error2 = _mm256_setzero_ps();
let mut error3 = _mm256_setzero_ps();
let mut error4 = _mm256_setzero_ps();
for y in 1..7 {
let left = unsafe { _mm256_maskload_ps(rows[y].as_ptr(), mask) };
let value = unsafe { _mm256_maskload_ps(rows[y].as_ptr().add(1), mask) };
let top = unsafe { _mm256_maskload_ps(rows[y - 1].as_ptr().add(1), mask) };
let top_left = unsafe { _mm256_maskload_ps(rows[y - 1].as_ptr(), mask) };
let bottom_left = unsafe { _mm256_maskload_ps(rows[y + 1].as_ptr(), mask) };
let gradient = _mm256_sub_ps(_mm256_add_ps(left, top), top_left);
let e0 = _mm256_sub_ps(value, left);
let e1 = _mm256_sub_ps(value, top);
let e2 = _mm256_sub_ps(value, top_left);
let e3 = _mm256_sub_ps(value, bottom_left);
let e4 = _mm256_sub_ps(value, gradient);
error0 = _mm256_fmadd_ps(e0, e0, error0);
error1 = _mm256_fmadd_ps(e1, e1, error1);
error2 = _mm256_fmadd_ps(e2, e2, error2);
error3 = _mm256_fmadd_ps(e3, e3, error3);
error4 = _mm256_fmadd_ps(e4, e4, error4);
}
[
hsum256(error0),
hsum256(error1),
hsum256(error2),
hsum256(error3),
hsum256(error4),
]
}
#[inline]
#[target_feature(enable = "avx2,fma")]
fn gradient_energy(values: &[f32], stride: usize, width: usize, height: usize) -> f32 {
let horizontal_mask = first_lanes_mask(width - 1);
let vertical_mask = first_lanes_mask(width);
let mut sum0 = _mm256_setzero_ps();
let mut sum1 = _mm256_setzero_ps();
let mut sum2 = _mm256_setzero_ps();
let mut sum3 = _mm256_setzero_ps();
for base in (0..height).step_by(4) {
let row0 = &values[base * stride..];
let row1 = &values[(base + 1) * stride..];
let row2 = &values[(base + 2) * stride..];
let row3 = &values[(base + 3) * stride..];
let d0 = _mm256_sub_ps(
unsafe { _mm256_maskload_ps(row0.as_ptr().add(1), horizontal_mask) },
unsafe { _mm256_maskload_ps(row0.as_ptr(), horizontal_mask) },
);
let d1 = _mm256_sub_ps(
unsafe { _mm256_maskload_ps(row1.as_ptr().add(1), horizontal_mask) },
unsafe { _mm256_maskload_ps(row1.as_ptr(), horizontal_mask) },
);
let d2 = _mm256_sub_ps(
unsafe { _mm256_maskload_ps(row2.as_ptr().add(1), horizontal_mask) },
unsafe { _mm256_maskload_ps(row2.as_ptr(), horizontal_mask) },
);
let d3 = _mm256_sub_ps(
unsafe { _mm256_maskload_ps(row3.as_ptr().add(1), horizontal_mask) },
unsafe { _mm256_maskload_ps(row3.as_ptr(), horizontal_mask) },
);
sum0 = _mm256_fmadd_ps(d0, d0, sum0);
sum1 = _mm256_fmadd_ps(d1, d1, sum1);
sum2 = _mm256_fmadd_ps(d2, d2, sum2);
sum3 = _mm256_fmadd_ps(d3, d3, sum3);
}
let vertical_rows = height - 1;
for base in (0..vertical_rows).step_by(4) {
let d0 = _mm256_sub_ps(
unsafe { _mm256_maskload_ps(values[(base + 1) * stride..].as_ptr(), vertical_mask) },
unsafe { _mm256_maskload_ps(values[base * stride..].as_ptr(), vertical_mask) },
);
sum0 = _mm256_fmadd_ps(d0, d0, sum0);
if base + 1 < vertical_rows {
let d1 = _mm256_sub_ps(
unsafe {
_mm256_maskload_ps(values[(base + 2) * stride..].as_ptr(), vertical_mask)
},
unsafe {
_mm256_maskload_ps(values[(base + 1) * stride..].as_ptr(), vertical_mask)
},
);
sum1 = _mm256_fmadd_ps(d1, d1, sum1);
}
if base + 2 < vertical_rows {
let d2 = _mm256_sub_ps(
unsafe {
_mm256_maskload_ps(values[(base + 3) * stride..].as_ptr(), vertical_mask)
},
unsafe {
_mm256_maskload_ps(values[(base + 2) * stride..].as_ptr(), vertical_mask)
},
);
sum2 = _mm256_fmadd_ps(d2, d2, sum2);
}
if base + 3 < vertical_rows {
let d3 = _mm256_sub_ps(
unsafe {
_mm256_maskload_ps(values[(base + 4) * stride..].as_ptr(), vertical_mask)
},
unsafe {
_mm256_maskload_ps(values[(base + 3) * stride..].as_ptr(), vertical_mask)
},
);
sum3 = _mm256_fmadd_ps(d3, d3, sum3);
}
}
hsum256(_mm256_add_ps(
_mm256_add_ps(sum0, sum1),
_mm256_add_ps(sum2, sum3),
))
}
#[inline]
#[target_feature(enable = "avx2,fma")]
fn downsample_8x8(block: &[f32; 64]) -> [f32; 16] {
let rows = block.as_chunks::<8>().0;
let mut half = [0.0; 16];
for (y, dst) in half.as_chunks_mut::<4>().0.iter_mut().enumerate() {
let top = unsafe { _mm256_loadu_ps(rows[2 * y].as_ptr()) };
let bottom = unsafe { _mm256_loadu_ps(rows[2 * y + 1].as_ptr()) };
let pairs = _mm256_hadd_ps(top, bottom);
let top_pairs = _mm256_permutevar8x32_ps(pairs, _mm256_setr_epi32(0, 1, 4, 5, 0, 0, 0, 0));
let bottom_pairs =
_mm256_permutevar8x32_ps(pairs, _mm256_setr_epi32(2, 3, 6, 7, 0, 0, 0, 0));
let value = _mm256_mul_ps(_mm256_add_ps(top_pairs, bottom_pairs), _mm256_set1_ps(0.25));
unsafe { _mm_storeu_ps(dst.as_mut_ptr(), _mm256_castps256_ps128(value)) };
}
half
}
#[inline]
#[target_feature(enable = "avx2,fma")]
fn energy_sum(values: &[f32]) -> f32 {
if values.is_empty() {
return 0.0;
}
let value = unsafe { _mm256_maskload_ps(values.as_ptr(), first_lanes_mask(values.len())) };
hsum256(_mm256_mul_ps(value, value))
}
#[inline]
#[target_feature(enable = "avx2,fma")]
fn spectral_energy(coeffs: &[f32; 64]) -> (f32, f32) {
let mut mid = 0.0;
let mut high = 0.0;
for (y, row) in coeffs.as_chunks::<8>().0.iter().enumerate() {
let mid_start = 2usize.saturating_sub(y);
let mid_end = (8 - y).min(8);
mid += energy_sum(&row[mid_start..mid_end]);
high += energy_sum(&row[8usize.saturating_sub(y)..]);
}
(mid, high)
}
#[target_feature(enable = "avx2,fma")]
pub(crate) fn block_features_avx2(block: &[f32; 64], dct8x8: &DctFn<8, 8, 64>) -> Features {
let (_, variance) = moments(block);
let tensor = tensor(block);
let errors = predictor_errors(block);
let energy_1x = gradient_energy(block, 8, 8, 8);
let half = downsample_8x8(block);
let energy_2x = gradient_energy(&half, 4, 4, 4);
let mut coeffs = [0.0; 64];
dct8x8(DctInput::from_flat(block), &mut coeffs);
let (mid, high) = spectral_energy(&coeffs);
finish_block_features(variance, tensor, errors, energy_1x, energy_2x, mid, high)
}