#![allow(dead_code)]
#[cfg(any(
target_arch = "aarch64",
target_arch = "x86_64",
all(target_arch = "wasm32", target_feature = "simd128")
))]
use crate::row::arch;
#[cfg(target_arch = "aarch64")]
use crate::row::neon_available;
#[cfg(target_arch = "x86_64")]
use crate::row::{avx2_available, avx512_available, sse41_available};
use crate::{
KernelMatrix,
row::{rgb_row_bytes, rgb_row_elems, rgba_row_bytes, rgba_row_elems, scalar},
};
#[cfg_attr(not(tarpaulin), inline(always))]
fn rgb96_packed_elems(width: usize) -> usize {
match width.checked_mul(3) {
Some(n) => n,
None => panic!("width ({width}) x 3 overflows usize (Rgb96 packed row)"),
}
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn rgb96_to_rgb_row_endian<const BE: bool>(
rgb96: &[u32],
rgb_out: &mut [u8],
width: usize,
use_simd: bool,
) {
let in_min = rgb96_packed_elems(width);
let out_min = rgb_row_bytes(width);
assert!(rgb96.len() >= in_min, "rgb96 row too short");
assert!(rgb_out.len() >= out_min, "rgb_out row too short");
if use_simd {
cfg_select! {
target_arch = "aarch64" => {
if neon_available() {
unsafe {
arch::neon::neon_rgb96_to_rgb_row::<BE>(rgb96, rgb_out, width);
}
return;
}
}
target_arch = "x86_64" => {
if avx512_available() {
unsafe {
arch::x86_avx512::avx512_rgb96_to_rgb_row::<BE>(rgb96, rgb_out, width);
}
return;
}
if avx2_available() {
unsafe {
arch::x86_avx2::avx2_rgb96_to_rgb_row::<BE>(rgb96, rgb_out, width);
}
return;
}
if sse41_available() {
unsafe {
arch::x86_sse41::sse41_rgb96_to_rgb_row::<BE>(rgb96, rgb_out, width);
}
return;
}
}
all(target_arch = "wasm32", target_feature = "simd128") => {
unsafe {
arch::wasm_simd128::wasm_rgb96_to_rgb_row::<BE>(rgb96, rgb_out, width);
}
return;
}
_ => {}
}
}
scalar::rgb96_to_rgb_row::<BE>(rgb96, rgb_out, width);
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn rgb96_to_rgb_row(rgb96: &[u32], rgb_out: &mut [u8], width: usize, use_simd: bool) {
rgb96_to_rgb_row_endian::<false>(rgb96, rgb_out, width, use_simd)
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn rgb96_to_rgba_row_endian<const BE: bool>(
rgb96: &[u32],
rgba_out: &mut [u8],
width: usize,
use_simd: bool,
) {
let in_min = rgb96_packed_elems(width);
let out_min = rgba_row_bytes(width);
assert!(rgb96.len() >= in_min, "rgb96 row too short");
assert!(rgba_out.len() >= out_min, "rgba_out row too short");
if use_simd {
cfg_select! {
target_arch = "aarch64" => {
if neon_available() {
unsafe {
arch::neon::neon_rgb96_to_rgba_row::<BE>(rgb96, rgba_out, width);
}
return;
}
}
target_arch = "x86_64" => {
if avx512_available() {
unsafe {
arch::x86_avx512::avx512_rgb96_to_rgba_row::<BE>(rgb96, rgba_out, width);
}
return;
}
if avx2_available() {
unsafe {
arch::x86_avx2::avx2_rgb96_to_rgba_row::<BE>(rgb96, rgba_out, width);
}
return;
}
if sse41_available() {
unsafe {
arch::x86_sse41::sse41_rgb96_to_rgba_row::<BE>(rgb96, rgba_out, width);
}
return;
}
}
all(target_arch = "wasm32", target_feature = "simd128") => {
unsafe {
arch::wasm_simd128::wasm_rgb96_to_rgba_row::<BE>(rgb96, rgba_out, width);
}
return;
}
_ => {}
}
}
scalar::rgb96_to_rgba_row::<BE>(rgb96, rgba_out, width);
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn rgb96_to_rgba_row(rgb96: &[u32], rgba_out: &mut [u8], width: usize, use_simd: bool) {
rgb96_to_rgba_row_endian::<false>(rgb96, rgba_out, width, use_simd)
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn rgb96_to_rgb_u16_row_endian<const BE: bool>(
rgb96: &[u32],
rgb_out: &mut [u16],
width: usize,
use_simd: bool,
) {
let in_min = rgb96_packed_elems(width);
let out_min = rgb_row_elems(width);
assert!(rgb96.len() >= in_min, "rgb96 row too short");
assert!(rgb_out.len() >= out_min, "rgb_out row too short");
if use_simd {
cfg_select! {
target_arch = "aarch64" => {
if neon_available() {
unsafe {
arch::neon::neon_rgb96_to_rgb_u16_row::<BE>(rgb96, rgb_out, width);
}
return;
}
}
target_arch = "x86_64" => {
if avx512_available() {
unsafe {
arch::x86_avx512::avx512_rgb96_to_rgb_u16_row::<BE>(rgb96, rgb_out, width);
}
return;
}
if avx2_available() {
unsafe {
arch::x86_avx2::avx2_rgb96_to_rgb_u16_row::<BE>(rgb96, rgb_out, width);
}
return;
}
if sse41_available() {
unsafe {
arch::x86_sse41::sse41_rgb96_to_rgb_u16_row::<BE>(rgb96, rgb_out, width);
}
return;
}
}
all(target_arch = "wasm32", target_feature = "simd128") => {
unsafe {
arch::wasm_simd128::wasm_rgb96_to_rgb_u16_row::<BE>(rgb96, rgb_out, width);
}
return;
}
_ => {}
}
}
scalar::rgb96_to_rgb_u16_row::<BE>(rgb96, rgb_out, width);
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn rgb96_to_rgb_u16_row(rgb96: &[u32], rgb_out: &mut [u16], width: usize, use_simd: bool) {
rgb96_to_rgb_u16_row_endian::<false>(rgb96, rgb_out, width, use_simd)
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn rgb96_to_rgba_u16_row_endian<const BE: bool>(
rgb96: &[u32],
rgba_out: &mut [u16],
width: usize,
use_simd: bool,
) {
let in_min = rgb96_packed_elems(width);
let out_min = rgba_row_elems(width);
assert!(rgb96.len() >= in_min, "rgb96 row too short");
assert!(rgba_out.len() >= out_min, "rgba_out row too short");
if use_simd {
cfg_select! {
target_arch = "aarch64" => {
if neon_available() {
unsafe {
arch::neon::neon_rgb96_to_rgba_u16_row::<BE>(rgb96, rgba_out, width);
}
return;
}
}
target_arch = "x86_64" => {
if avx512_available() {
unsafe {
arch::x86_avx512::avx512_rgb96_to_rgba_u16_row::<BE>(rgb96, rgba_out, width);
}
return;
}
if avx2_available() {
unsafe {
arch::x86_avx2::avx2_rgb96_to_rgba_u16_row::<BE>(rgb96, rgba_out, width);
}
return;
}
if sse41_available() {
unsafe {
arch::x86_sse41::sse41_rgb96_to_rgba_u16_row::<BE>(rgb96, rgba_out, width);
}
return;
}
}
all(target_arch = "wasm32", target_feature = "simd128") => {
unsafe {
arch::wasm_simd128::wasm_rgb96_to_rgba_u16_row::<BE>(rgb96, rgba_out, width);
}
return;
}
_ => {}
}
}
scalar::rgb96_to_rgba_u16_row::<BE>(rgb96, rgba_out, width);
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn rgb96_to_rgba_u16_row(rgb96: &[u32], rgba_out: &mut [u16], width: usize, use_simd: bool) {
rgb96_to_rgba_u16_row_endian::<false>(rgb96, rgba_out, width, use_simd)
}
#[cfg_attr(not(tarpaulin), inline(always))]
#[allow(clippy::too_many_arguments)]
pub fn rgb96_to_luma_row_endian<const BE: bool>(
rgb96: &[u32],
luma_out: &mut [u8],
rgb_scratch: &mut [u8],
width: usize,
matrix: KernelMatrix,
full_range: bool,
use_simd: bool,
) {
let in_min = rgb96_packed_elems(width);
let scratch_min = rgb_row_bytes(width);
assert!(rgb96.len() >= in_min, "rgb96 row too short");
assert!(rgb_scratch.len() >= scratch_min, "rgb_scratch too short");
assert!(luma_out.len() >= width, "luma_out row too short");
rgb96_to_rgb_row_endian::<BE>(rgb96, rgb_scratch, width, use_simd);
scalar::rgb_to_luma_row(rgb_scratch, luma_out, width, matrix, full_range);
}
#[allow(clippy::too_many_arguments)]
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn rgb96_to_luma_row(
rgb96: &[u32],
luma_out: &mut [u8],
rgb_scratch: &mut [u8],
width: usize,
matrix: KernelMatrix,
full_range: bool,
use_simd: bool,
) {
rgb96_to_luma_row_endian::<false>(
rgb96,
luma_out,
rgb_scratch,
width,
matrix,
full_range,
use_simd,
)
}
#[cfg_attr(not(tarpaulin), inline(always))]
#[allow(clippy::too_many_arguments)]
pub fn rgb96_to_luma_u16_row_endian<const BE: bool>(
rgb96: &[u32],
luma_out: &mut [u16],
rgb_scratch: &mut [u8],
width: usize,
matrix: KernelMatrix,
full_range: bool,
use_simd: bool,
) {
let in_min = rgb96_packed_elems(width);
let scratch_min = rgb_row_bytes(width);
assert!(rgb96.len() >= in_min, "rgb96 row too short");
assert!(rgb_scratch.len() >= scratch_min, "rgb_scratch too short");
assert!(luma_out.len() >= width, "luma_out row too short");
rgb96_to_rgb_row_endian::<BE>(rgb96, rgb_scratch, width, use_simd);
scalar::rgb_to_luma_u16_row(rgb_scratch, luma_out, width, matrix, full_range);
}
#[allow(clippy::too_many_arguments)]
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn rgb96_to_luma_u16_row(
rgb96: &[u32],
luma_out: &mut [u16],
rgb_scratch: &mut [u8],
width: usize,
matrix: KernelMatrix,
full_range: bool,
use_simd: bool,
) {
rgb96_to_luma_u16_row_endian::<false>(
rgb96,
luma_out,
rgb_scratch,
width,
matrix,
full_range,
use_simd,
)
}
#[cfg_attr(not(tarpaulin), inline(always))]
#[allow(clippy::too_many_arguments)]
pub fn rgb96_to_hsv_row_endian<const BE: bool>(
rgb96: &[u32],
h_out: &mut [u8],
s_out: &mut [u8],
v_out: &mut [u8],
rgb_scratch: &mut [u8],
width: usize,
use_simd: bool,
) {
let in_min = rgb96_packed_elems(width);
let scratch_min = rgb_row_bytes(width);
assert!(rgb96.len() >= in_min, "rgb96 row too short");
assert!(rgb_scratch.len() >= scratch_min, "rgb_scratch too short");
assert!(h_out.len() >= width, "h_out row too short");
assert!(s_out.len() >= width, "s_out row too short");
assert!(v_out.len() >= width, "v_out row too short");
rgb96_to_rgb_row_endian::<BE>(rgb96, rgb_scratch, width, use_simd);
scalar::rgb_to_hsv_row(rgb_scratch, h_out, s_out, v_out, width);
}
#[allow(clippy::too_many_arguments)]
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn rgb96_to_hsv_row(
rgb96: &[u32],
h_out: &mut [u8],
s_out: &mut [u8],
v_out: &mut [u8],
rgb_scratch: &mut [u8],
width: usize,
use_simd: bool,
) {
rgb96_to_hsv_row_endian::<false>(rgb96, h_out, s_out, v_out, rgb_scratch, width, use_simd)
}
#[cfg_attr(not(tarpaulin), inline(always))]
fn rgba128_packed_elems(width: usize) -> usize {
match width.checked_mul(4) {
Some(n) => n,
None => panic!("width ({width}) x 4 overflows usize (Rgba128 packed row)"),
}
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn rgba128_to_rgb_row_endian<const BE: bool>(
rgba128: &[u32],
rgb_out: &mut [u8],
width: usize,
use_simd: bool,
) {
let in_min = rgba128_packed_elems(width);
let out_min = rgb_row_bytes(width);
assert!(rgba128.len() >= in_min, "rgba128 row too short");
assert!(rgb_out.len() >= out_min, "rgb_out row too short");
if use_simd {
cfg_select! {
target_arch = "aarch64" => {
if neon_available() {
unsafe {
arch::neon::neon_rgba128_to_rgb_row::<BE>(rgba128, rgb_out, width);
}
return;
}
}
target_arch = "x86_64" => {
if avx512_available() {
unsafe {
arch::x86_avx512::avx512_rgba128_to_rgb_row::<BE>(rgba128, rgb_out, width);
}
return;
}
if avx2_available() {
unsafe {
arch::x86_avx2::avx2_rgba128_to_rgb_row::<BE>(rgba128, rgb_out, width);
}
return;
}
if sse41_available() {
unsafe {
arch::x86_sse41::sse41_rgba128_to_rgb_row::<BE>(rgba128, rgb_out, width);
}
return;
}
}
all(target_arch = "wasm32", target_feature = "simd128") => {
unsafe {
arch::wasm_simd128::wasm_rgba128_to_rgb_row::<BE>(rgba128, rgb_out, width);
}
return;
}
_ => {}
}
}
scalar::rgba128_to_rgb_row::<BE>(rgba128, rgb_out, width);
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn rgba128_to_rgb_row(rgba128: &[u32], rgb_out: &mut [u8], width: usize, use_simd: bool) {
rgba128_to_rgb_row_endian::<false>(rgba128, rgb_out, width, use_simd)
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn rgba128_to_rgba_row_endian<const BE: bool>(
rgba128: &[u32],
rgba_out: &mut [u8],
width: usize,
use_simd: bool,
) {
let in_min = rgba128_packed_elems(width);
let out_min = rgba_row_bytes(width);
assert!(rgba128.len() >= in_min, "rgba128 row too short");
assert!(rgba_out.len() >= out_min, "rgba_out row too short");
if use_simd {
cfg_select! {
target_arch = "aarch64" => {
if neon_available() {
unsafe {
arch::neon::neon_rgba128_to_rgba_row::<BE>(rgba128, rgba_out, width);
}
return;
}
}
target_arch = "x86_64" => {
if avx512_available() {
unsafe {
arch::x86_avx512::avx512_rgba128_to_rgba_row::<BE>(rgba128, rgba_out, width);
}
return;
}
if avx2_available() {
unsafe {
arch::x86_avx2::avx2_rgba128_to_rgba_row::<BE>(rgba128, rgba_out, width);
}
return;
}
if sse41_available() {
unsafe {
arch::x86_sse41::sse41_rgba128_to_rgba_row::<BE>(rgba128, rgba_out, width);
}
return;
}
}
all(target_arch = "wasm32", target_feature = "simd128") => {
unsafe {
arch::wasm_simd128::wasm_rgba128_to_rgba_row::<BE>(rgba128, rgba_out, width);
}
return;
}
_ => {}
}
}
scalar::rgba128_to_rgba_row::<BE>(rgba128, rgba_out, width);
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn rgba128_to_rgba_row(rgba128: &[u32], rgba_out: &mut [u8], width: usize, use_simd: bool) {
rgba128_to_rgba_row_endian::<false>(rgba128, rgba_out, width, use_simd)
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn rgba128_to_rgb_u16_row_endian<const BE: bool>(
rgba128: &[u32],
rgb_out: &mut [u16],
width: usize,
use_simd: bool,
) {
let in_min = rgba128_packed_elems(width);
let out_min = rgb_row_elems(width);
assert!(rgba128.len() >= in_min, "rgba128 row too short");
assert!(rgb_out.len() >= out_min, "rgb_out row too short");
if use_simd {
cfg_select! {
target_arch = "aarch64" => {
if neon_available() {
unsafe {
arch::neon::neon_rgba128_to_rgb_u16_row::<BE>(rgba128, rgb_out, width);
}
return;
}
}
target_arch = "x86_64" => {
if avx512_available() {
unsafe {
arch::x86_avx512::avx512_rgba128_to_rgb_u16_row::<BE>(rgba128, rgb_out, width);
}
return;
}
if avx2_available() {
unsafe {
arch::x86_avx2::avx2_rgba128_to_rgb_u16_row::<BE>(rgba128, rgb_out, width);
}
return;
}
if sse41_available() {
unsafe {
arch::x86_sse41::sse41_rgba128_to_rgb_u16_row::<BE>(rgba128, rgb_out, width);
}
return;
}
}
all(target_arch = "wasm32", target_feature = "simd128") => {
unsafe {
arch::wasm_simd128::wasm_rgba128_to_rgb_u16_row::<BE>(rgba128, rgb_out, width);
}
return;
}
_ => {}
}
}
scalar::rgba128_to_rgb_u16_row::<BE>(rgba128, rgb_out, width);
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn rgba128_to_rgb_u16_row(rgba128: &[u32], rgb_out: &mut [u16], width: usize, use_simd: bool) {
rgba128_to_rgb_u16_row_endian::<false>(rgba128, rgb_out, width, use_simd)
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn rgba128_to_rgba_u16_row_endian<const BE: bool>(
rgba128: &[u32],
rgba_out: &mut [u16],
width: usize,
use_simd: bool,
) {
let in_min = rgba128_packed_elems(width);
let out_min = rgba_row_elems(width);
assert!(rgba128.len() >= in_min, "rgba128 row too short");
assert!(rgba_out.len() >= out_min, "rgba_out row too short");
if use_simd {
cfg_select! {
target_arch = "aarch64" => {
if neon_available() {
unsafe {
arch::neon::neon_rgba128_to_rgba_u16_row::<BE>(rgba128, rgba_out, width);
}
return;
}
}
target_arch = "x86_64" => {
if avx512_available() {
unsafe {
arch::x86_avx512::avx512_rgba128_to_rgba_u16_row::<BE>(rgba128, rgba_out, width);
}
return;
}
if avx2_available() {
unsafe {
arch::x86_avx2::avx2_rgba128_to_rgba_u16_row::<BE>(rgba128, rgba_out, width);
}
return;
}
if sse41_available() {
unsafe {
arch::x86_sse41::sse41_rgba128_to_rgba_u16_row::<BE>(rgba128, rgba_out, width);
}
return;
}
}
all(target_arch = "wasm32", target_feature = "simd128") => {
unsafe {
arch::wasm_simd128::wasm_rgba128_to_rgba_u16_row::<BE>(rgba128, rgba_out, width);
}
return;
}
_ => {}
}
}
scalar::rgba128_to_rgba_u16_row::<BE>(rgba128, rgba_out, width);
}
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn rgba128_to_rgba_u16_row(
rgba128: &[u32],
rgba_out: &mut [u16],
width: usize,
use_simd: bool,
) {
rgba128_to_rgba_u16_row_endian::<false>(rgba128, rgba_out, width, use_simd)
}
#[cfg_attr(not(tarpaulin), inline(always))]
#[allow(clippy::too_many_arguments)]
pub fn rgba128_to_luma_row_endian<const BE: bool>(
rgba128: &[u32],
luma_out: &mut [u8],
rgb_scratch: &mut [u8],
width: usize,
matrix: KernelMatrix,
full_range: bool,
use_simd: bool,
) {
let in_min = rgba128_packed_elems(width);
let scratch_min = rgb_row_bytes(width);
assert!(rgba128.len() >= in_min, "rgba128 row too short");
assert!(rgb_scratch.len() >= scratch_min, "rgb_scratch too short");
assert!(luma_out.len() >= width, "luma_out row too short");
rgba128_to_rgb_row_endian::<BE>(rgba128, rgb_scratch, width, use_simd);
scalar::rgb_to_luma_row(rgb_scratch, luma_out, width, matrix, full_range);
}
#[allow(clippy::too_many_arguments)]
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn rgba128_to_luma_row(
rgba128: &[u32],
luma_out: &mut [u8],
rgb_scratch: &mut [u8],
width: usize,
matrix: KernelMatrix,
full_range: bool,
use_simd: bool,
) {
rgba128_to_luma_row_endian::<false>(
rgba128,
luma_out,
rgb_scratch,
width,
matrix,
full_range,
use_simd,
)
}
#[cfg_attr(not(tarpaulin), inline(always))]
#[allow(clippy::too_many_arguments)]
pub fn rgba128_to_luma_u16_row_endian<const BE: bool>(
rgba128: &[u32],
luma_out: &mut [u16],
rgb_scratch: &mut [u8],
width: usize,
matrix: KernelMatrix,
full_range: bool,
use_simd: bool,
) {
let in_min = rgba128_packed_elems(width);
let scratch_min = rgb_row_bytes(width);
assert!(rgba128.len() >= in_min, "rgba128 row too short");
assert!(rgb_scratch.len() >= scratch_min, "rgb_scratch too short");
assert!(luma_out.len() >= width, "luma_out row too short");
rgba128_to_rgb_row_endian::<BE>(rgba128, rgb_scratch, width, use_simd);
scalar::rgb_to_luma_u16_row(rgb_scratch, luma_out, width, matrix, full_range);
}
#[allow(clippy::too_many_arguments)]
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn rgba128_to_luma_u16_row(
rgba128: &[u32],
luma_out: &mut [u16],
rgb_scratch: &mut [u8],
width: usize,
matrix: KernelMatrix,
full_range: bool,
use_simd: bool,
) {
rgba128_to_luma_u16_row_endian::<false>(
rgba128,
luma_out,
rgb_scratch,
width,
matrix,
full_range,
use_simd,
)
}
#[cfg_attr(not(tarpaulin), inline(always))]
#[allow(clippy::too_many_arguments)]
pub fn rgba128_to_hsv_row_endian<const BE: bool>(
rgba128: &[u32],
h_out: &mut [u8],
s_out: &mut [u8],
v_out: &mut [u8],
rgb_scratch: &mut [u8],
width: usize,
use_simd: bool,
) {
let in_min = rgba128_packed_elems(width);
let scratch_min = rgb_row_bytes(width);
assert!(rgba128.len() >= in_min, "rgba128 row too short");
assert!(rgb_scratch.len() >= scratch_min, "rgb_scratch too short");
assert!(h_out.len() >= width, "h_out row too short");
assert!(s_out.len() >= width, "s_out row too short");
assert!(v_out.len() >= width, "v_out row too short");
rgba128_to_rgb_row_endian::<BE>(rgba128, rgb_scratch, width, use_simd);
scalar::rgb_to_hsv_row(rgb_scratch, h_out, s_out, v_out, width);
}
#[allow(clippy::too_many_arguments)]
#[cfg_attr(not(tarpaulin), inline(always))]
pub fn rgba128_to_hsv_row(
rgba128: &[u32],
h_out: &mut [u8],
s_out: &mut [u8],
v_out: &mut [u8],
rgb_scratch: &mut [u8],
width: usize,
use_simd: bool,
) {
rgba128_to_hsv_row_endian::<false>(rgba128, h_out, s_out, v_out, rgb_scratch, width, use_simd)
}