#![cfg_attr(
any(
not(feature = "std"),
not(any(feature = "yuv-planar", feature = "rgb"))
),
allow(dead_code)
)]
#[cfg(any(
target_arch = "aarch64",
target_arch = "x86_64",
target_arch = "wasm32"
))]
use crate::row::arch;
#[cfg(target_arch = "aarch64")]
use crate::row::neon_available;
use std::vec::Vec;
use crate::row::scalar::area_reduce as scalar;
#[cfg(target_arch = "wasm32")]
use crate::row::simd128_available;
#[cfg(target_arch = "x86_64")]
use crate::row::{avx2_available, avx512_available, sse41_available};
#[derive(Debug)]
pub(crate) struct PaddedSpans {
starts: Vec<usize>,
w16: Vec<u16>,
off: Vec<usize>,
max_reach: usize,
}
impl PaddedSpans {
pub(crate) fn build(starts: &[usize], offsets: &[usize], weights: &[usize]) -> Option<Self> {
let out = starts.len();
if offsets.len() != out + 1 || out > usize::from(u16::MAX) {
return None;
}
let mut total = 0usize;
let mut max_reach = 0usize;
for j in 0..out {
let k = offsets[j + 1].checked_sub(offsets[j])?;
let k_pad = k.div_ceil(8).checked_mul(8)?;
total = total.checked_add(k_pad)?;
max_reach = max_reach.max(starts[j].checked_add(k)?);
}
if weights.len() < offsets[out] {
return None;
}
let mut w16 = Vec::new();
let mut off = Vec::new();
let mut starts_own = Vec::new();
if w16.try_reserve_exact(total).is_err()
|| off.try_reserve_exact(out + 1).is_err()
|| starts_own.try_reserve_exact(out).is_err()
{
return None;
}
starts_own.extend_from_slice(starts);
off.push(0);
for j in 0..out {
let span = &weights[offsets[j]..offsets[j + 1]];
for &w in span {
w16.push(u16::try_from(w).ok()?);
}
w16.resize(w16.len() + (span.len().div_ceil(8) * 8 - span.len()), 0);
off.push(w16.len());
}
Some(Self {
starts: starts_own,
w16,
off,
max_reach,
})
}
}
#[cfg_attr(not(tarpaulin), inline(always))]
#[allow(clippy::too_many_arguments)]
pub(crate) fn area_h_reduce_row(
row: &[u8],
channels: usize,
starts: &[usize],
offsets: &[usize],
weights: &[usize],
padded: Option<&PaddedSpans>,
h_tmp: &mut [u32],
use_simd: bool,
) {
let out = starts.len();
assert!(offsets.len() == out + 1, "offsets shape");
assert!(h_tmp.len() >= out * channels, "h_tmp too short");
assert!(weights.len() >= offsets[out], "weights arena too short");
if use_simd && let Some(p) = padded {
assert!(p.starts.len() == out, "padded arena shape");
assert!(
p.off.len() == out + 1 && p.off[out] <= p.w16.len(),
"padded arena layout"
);
assert!(
p.max_reach <= row.len() / channels,
"padded arena exceeds row"
);
cfg_select! {
target_arch = "aarch64" => {
if neon_available() && channels == 1 {
unsafe {
arch::neon::area_reduce::area_h_reduce_row_c1(row, &p.starts, &p.w16, &p.off, h_tmp);
}
return;
}
if neon_available() && channels == 3 {
unsafe {
arch::neon::area_reduce::area_h_reduce_row_c3(row, &p.starts, &p.w16, &p.off, h_tmp);
}
return;
}
}
target_arch = "x86_64" => {
if channels == 1 {
if avx512_available() {
unsafe {
arch::x86_avx512::area_reduce::area_h_reduce_row_c1(
row, &p.starts, &p.w16, &p.off, h_tmp,
);
}
return;
}
if avx2_available() {
unsafe {
arch::x86_avx2::area_reduce::area_h_reduce_row_c1(
row, &p.starts, &p.w16, &p.off, h_tmp,
);
}
return;
}
if sse41_available() {
unsafe {
arch::x86_sse41::area_reduce::area_h_reduce_row_c1(
row, &p.starts, &p.w16, &p.off, h_tmp,
);
}
return;
}
}
if channels == 3 {
if avx512_available() {
unsafe {
arch::x86_avx512::area_reduce::area_h_reduce_row_c3(
row, &p.starts, &p.w16, &p.off, h_tmp,
);
}
return;
}
if avx2_available() {
unsafe {
arch::x86_avx2::area_reduce::area_h_reduce_row_c3(
row, &p.starts, &p.w16, &p.off, h_tmp,
);
}
return;
}
if sse41_available() {
unsafe {
arch::x86_sse41::area_reduce::area_h_reduce_row_c3(
row, &p.starts, &p.w16, &p.off, h_tmp,
);
}
return;
}
}
}
target_arch = "wasm32" => {
if simd128_available() && channels == 1 {
unsafe {
arch::wasm_simd128::area_reduce::area_h_reduce_row_c1(
row, &p.starts, &p.w16, &p.off, h_tmp,
);
}
return;
}
if simd128_available() && channels == 3 {
unsafe {
arch::wasm_simd128::area_reduce::area_h_reduce_row_c3(
row, &p.starts, &p.w16, &p.off, h_tmp,
);
}
return;
}
}
_ => {}
}
}
scalar::area_h_reduce_row(row, channels, starts, offsets, weights, h_tmp);
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub(crate) fn area_v_accumulate(acc: &mut [u64], h_tmp: &[u32], w: u64, use_simd: bool) {
assert!(h_tmp.len() >= acc.len(), "h_tmp too short");
if !use_simd {
return scalar::area_v_accumulate(acc, h_tmp, w);
}
#[cfg(any(
target_arch = "aarch64",
target_arch = "x86_64",
target_arch = "wasm32"
))]
if let Ok(w32) = u32::try_from(w) {
cfg_select! {
target_arch = "aarch64" => {
if neon_available() {
unsafe {
arch::neon::area_reduce::area_v_accumulate(acc, h_tmp, w32);
}
return;
}
}
target_arch = "x86_64" => {
if avx512_available() {
unsafe {
arch::x86_avx512::area_reduce::area_v_accumulate(acc, h_tmp, w32);
}
return;
}
if avx2_available() {
unsafe {
arch::x86_avx2::area_reduce::area_v_accumulate(acc, h_tmp, w32);
}
return;
}
if sse41_available() {
unsafe {
arch::x86_sse41::area_reduce::area_v_accumulate(acc, h_tmp, w32);
}
return;
}
}
target_arch = "wasm32" => {
if simd128_available() {
unsafe {
arch::wasm_simd128::area_reduce::area_v_accumulate(acc, h_tmp, w32);
}
return;
}
}
_ => {}
}
}
scalar::area_v_accumulate(acc, h_tmp, w);
}
#[cfg_attr(not(tarpaulin), inline(always))]
#[allow(clippy::too_many_arguments)]
pub(crate) fn area_h_reduce_row_u16(
row: &[u16],
channels: usize,
starts: &[usize],
offsets: &[usize],
weights: &[usize],
padded: Option<&PaddedSpans>,
h_tmp: &mut [u64],
use_simd: bool,
) {
let out = starts.len();
assert!(offsets.len() == out + 1, "offsets shape");
assert!(h_tmp.len() >= out * channels, "h_tmp too short");
assert!(weights.len() >= offsets[out], "weights arena too short");
if use_simd && let Some(p) = padded {
assert!(p.starts.len() == out, "padded arena shape");
assert!(
p.off.len() == out + 1 && p.off[out] <= p.w16.len(),
"padded arena layout"
);
assert!(
p.max_reach <= row.len() / channels,
"padded arena exceeds row"
);
cfg_select! {
target_arch = "aarch64" => {
if neon_available() && channels == 1 {
unsafe {
arch::neon::area_reduce::area_h_reduce_row_u16_c1(
row, &p.starts, &p.w16, &p.off, h_tmp,
);
}
return;
}
if neon_available() && channels == 3 {
unsafe {
arch::neon::area_reduce::area_h_reduce_row_u16_c3(
row, &p.starts, &p.w16, &p.off, h_tmp,
);
}
return;
}
}
target_arch = "x86_64" => {
if channels == 1 {
if avx512_available() {
unsafe {
arch::x86_avx512::area_reduce::area_h_reduce_row_u16_c1(
row, &p.starts, &p.w16, &p.off, h_tmp,
);
}
return;
}
if avx2_available() {
unsafe {
arch::x86_avx2::area_reduce::area_h_reduce_row_u16_c1(
row, &p.starts, &p.w16, &p.off, h_tmp,
);
}
return;
}
if sse41_available() {
unsafe {
arch::x86_sse41::area_reduce::area_h_reduce_row_u16_c1(
row, &p.starts, &p.w16, &p.off, h_tmp,
);
}
return;
}
}
if channels == 3 {
if avx512_available() {
unsafe {
arch::x86_avx512::area_reduce::area_h_reduce_row_u16_c3(
row, &p.starts, &p.w16, &p.off, h_tmp,
);
}
return;
}
if avx2_available() {
unsafe {
arch::x86_avx2::area_reduce::area_h_reduce_row_u16_c3(
row, &p.starts, &p.w16, &p.off, h_tmp,
);
}
return;
}
if sse41_available() {
unsafe {
arch::x86_sse41::area_reduce::area_h_reduce_row_u16_c3(
row, &p.starts, &p.w16, &p.off, h_tmp,
);
}
return;
}
}
}
target_arch = "wasm32" => {
if simd128_available() && channels == 1 {
unsafe {
arch::wasm_simd128::area_reduce::area_h_reduce_row_u16_c1(
row, &p.starts, &p.w16, &p.off, h_tmp,
);
}
return;
}
if simd128_available() && channels == 3 {
unsafe {
arch::wasm_simd128::area_reduce::area_h_reduce_row_u16_c3(
row, &p.starts, &p.w16, &p.off, h_tmp,
);
}
return;
}
}
_ => {}
}
}
scalar::area_h_reduce_row_u16(row, channels, starts, offsets, weights, h_tmp);
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub(crate) fn area_v_accumulate_u16(acc: &mut [u64], h_tmp: &[u64], w: u64, use_simd: bool) {
assert!(h_tmp.len() >= acc.len(), "h_tmp too short");
if !use_simd {
return scalar::area_v_accumulate_u16(acc, h_tmp, w);
}
#[cfg(any(
target_arch = "aarch64",
target_arch = "x86_64",
target_arch = "wasm32"
))]
if let Ok(w32) = u32::try_from(w) {
cfg_select! {
target_arch = "aarch64" => {
if neon_available() {
unsafe {
arch::neon::area_reduce::area_v_accumulate_u16(acc, h_tmp, w32);
}
return;
}
}
target_arch = "x86_64" => {
if avx512_available() {
unsafe {
arch::x86_avx512::area_reduce::area_v_accumulate_u16(acc, h_tmp, w32);
}
return;
}
if avx2_available() {
unsafe {
arch::x86_avx2::area_reduce::area_v_accumulate_u16(acc, h_tmp, w32);
}
return;
}
if sse41_available() {
unsafe {
arch::x86_sse41::area_reduce::area_v_accumulate_u16(acc, h_tmp, w32);
}
return;
}
}
target_arch = "wasm32" => {
if simd128_available() {
unsafe {
arch::wasm_simd128::area_reduce::area_v_accumulate_u16(acc, h_tmp, w32);
}
return;
}
}
_ => {}
}
}
scalar::area_v_accumulate_u16(acc, h_tmp, w);
}
#[cfg_attr(not(tarpaulin), inline(always))]
#[allow(clippy::too_many_arguments)]
pub(crate) fn area_h_reduce_row_f32(
row: &[f32],
channels: usize,
starts: &[usize],
offsets: &[usize],
weights: &[usize],
padded: Option<&PaddedSpans>,
h_tmp: &mut [f64],
use_simd: bool,
) {
let out = starts.len();
assert!(offsets.len() == out + 1, "offsets shape");
assert!(h_tmp.len() >= out * channels, "h_tmp too short");
assert!(weights.len() >= offsets[out], "weights arena too short");
if use_simd && let Some(p) = padded {
assert!(p.starts.len() == out, "padded arena shape");
assert!(
p.off.len() == out + 1 && p.off[out] <= p.w16.len(),
"padded arena layout"
);
assert!(
p.max_reach <= row.len() / channels,
"padded arena exceeds row"
);
cfg_select! {
target_arch = "aarch64" => {
if neon_available() && channels == 1 {
unsafe {
arch::neon::area_reduce::area_h_reduce_row_f32_c1(
row, &p.starts, &p.w16, &p.off, h_tmp,
);
}
return;
}
if neon_available() && channels == 3 {
unsafe {
arch::neon::area_reduce::area_h_reduce_row_f32_c3(
row, &p.starts, &p.w16, &p.off, h_tmp,
);
}
return;
}
}
target_arch = "x86_64" => {
if channels == 1 {
if avx512_available() {
unsafe {
arch::x86_avx512::area_reduce::area_h_reduce_row_f32_c1(
row, &p.starts, &p.w16, &p.off, h_tmp,
);
}
return;
}
if avx2_available() {
unsafe {
arch::x86_avx2::area_reduce::area_h_reduce_row_f32_c1(
row, &p.starts, &p.w16, &p.off, h_tmp,
);
}
return;
}
if sse41_available() {
unsafe {
arch::x86_sse41::area_reduce::area_h_reduce_row_f32_c1(
row, &p.starts, &p.w16, &p.off, h_tmp,
);
}
return;
}
}
if channels == 3 {
if avx512_available() {
unsafe {
arch::x86_avx512::area_reduce::area_h_reduce_row_f32_c3(
row, &p.starts, &p.w16, &p.off, h_tmp,
);
}
return;
}
if avx2_available() {
unsafe {
arch::x86_avx2::area_reduce::area_h_reduce_row_f32_c3(
row, &p.starts, &p.w16, &p.off, h_tmp,
);
}
return;
}
if sse41_available() {
unsafe {
arch::x86_sse41::area_reduce::area_h_reduce_row_f32_c3(
row, &p.starts, &p.w16, &p.off, h_tmp,
);
}
return;
}
}
}
target_arch = "wasm32" => {
if simd128_available() && channels == 1 {
unsafe {
arch::wasm_simd128::area_reduce::area_h_reduce_row_f32_c1(
row, &p.starts, &p.w16, &p.off, h_tmp,
);
}
return;
}
if simd128_available() && channels == 3 {
unsafe {
arch::wasm_simd128::area_reduce::area_h_reduce_row_f32_c3(
row, &p.starts, &p.w16, &p.off, h_tmp,
);
}
return;
}
}
_ => {}
}
}
scalar::area_h_reduce_row_f32(row, channels, starts, offsets, weights, h_tmp);
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub(crate) fn area_v_accumulate_f32(acc: &mut [f64], h_tmp: &[f64], w: f64, use_simd: bool) {
assert!(h_tmp.len() >= acc.len(), "h_tmp too short");
if !use_simd {
return scalar::area_v_accumulate_f32(acc, h_tmp, w);
}
#[cfg(any(
target_arch = "aarch64",
target_arch = "x86_64",
target_arch = "wasm32"
))]
{
cfg_select! {
target_arch = "aarch64" => {
if neon_available() {
unsafe {
arch::neon::area_reduce::area_v_accumulate_f32(acc, h_tmp, w);
}
return;
}
}
target_arch = "x86_64" => {
if avx512_available() {
unsafe {
arch::x86_avx512::area_reduce::area_v_accumulate_f32(acc, h_tmp, w);
}
return;
}
if avx2_available() {
unsafe {
arch::x86_avx2::area_reduce::area_v_accumulate_f32(acc, h_tmp, w);
}
return;
}
if sse41_available() {
unsafe {
arch::x86_sse41::area_reduce::area_v_accumulate_f32(acc, h_tmp, w);
}
return;
}
}
target_arch = "wasm32" => {
if simd128_available() {
unsafe {
arch::wasm_simd128::area_reduce::area_v_accumulate_f32(acc, h_tmp, w);
}
return;
}
}
_ => {}
}
}
scalar::area_v_accumulate_f32(acc, h_tmp, w);
}
#[cfg(all(test, feature = "std"))]
mod tests {
use super::*;
fn arena(starts: &[usize], offsets: &[usize], weights: &[usize]) -> PaddedSpans {
PaddedSpans::build(starts, offsets, weights).expect("valid spans")
}
#[test]
fn build_proves_padding_and_reach() {
let p = arena(&[0, 4], &[0, 4, 13], &[1usize; 13]);
assert_eq!(p.off, std::vec![0, 8, 24]);
assert_eq!(p.w16.len(), 24);
assert_eq!(p.max_reach, 13);
assert_eq!(p.starts, std::vec![0, 4]);
assert_eq!(&p.w16[..4], [1, 1, 1, 1]);
assert_eq!(&p.w16[4..8], [0, 0, 0, 0]);
}
#[test]
fn build_rejects_an_overweight_weight() {
assert!(PaddedSpans::build(&[0], &[0, 1], &[65_536]).is_none());
}
#[test]
fn build_rejects_outputs_past_the_u16_weight_bound() {
let out = usize::from(u16::MAX) + 1;
let starts = std::vec![0usize; out];
let offsets: std::vec::Vec<usize> = (0..=out).collect();
let weights = std::vec![1usize; out];
assert!(PaddedSpans::build(&starts, &offsets, &weights).is_none());
}
#[test]
#[should_panic(expected = "padded arena shape")]
fn dispatcher_panics_on_plan_mismatch() {
let p = arena(&[0], &[0, 4], &[1usize; 4]);
let row = [0u8; 16];
let weights = [1usize; 8];
let mut h_tmp = [0u32; 2];
area_h_reduce_row(
&row,
1,
&[0, 4],
&[0, 4, 8],
&weights,
Some(&p),
&mut h_tmp,
true,
);
}
#[test]
#[should_panic(expected = "padded arena exceeds row")]
fn dispatcher_panics_on_row_shorter_than_reach() {
let p = arena(&[0], &[0, 4], &[1usize; 4]);
let row = [0u8; 3];
let weights = [1usize; 4];
let mut h_tmp = [0u32; 1];
area_h_reduce_row(&row, 1, &[0], &[0, 4], &weights, Some(&p), &mut h_tmp, true);
}
}