#![cfg_attr(not(feature = "std"), allow(dead_code))]
#[cfg_attr(miri, allow(unused_imports))]
use core::arch::x86_64::*;
use crate::row::{
arch::x86_sse41::endian::{load_endian_u16x4, load_endian_u16x8, load_endian_u32x4},
scalar::{bits_mask, gray as scalar},
};
#[allow(dead_code)]
#[inline]
#[target_feature(enable = "sse4.1")]
pub(crate) unsafe fn gray8_to_rgb_row(
y_plane: &[u8],
out: &mut [u8],
width: usize,
full_range: bool,
) {
debug_assert!(y_plane.len() >= width);
debug_assert!(out.len() >= width * 3);
scalar::gray8_to_rgb_row(y_plane, out, width, full_range);
}
#[allow(dead_code)]
#[inline]
#[target_feature(enable = "sse4.1")]
pub(crate) unsafe fn gray8_to_rgba_row(
y_plane: &[u8],
out: &mut [u8],
width: usize,
full_range: bool,
) {
debug_assert!(y_plane.len() >= width);
debug_assert!(out.len() >= width * 4);
scalar::gray8_to_rgba_row(y_plane, out, width, full_range);
}
#[allow(dead_code)]
#[inline]
#[target_feature(enable = "sse4.1")]
pub(crate) unsafe fn gray8_to_hsv_row(
y_plane: &[u8],
h_out: &mut [u8],
s_out: &mut [u8],
v_out: &mut [u8],
width: usize,
full_range: bool,
) {
debug_assert!(y_plane.len() >= width);
if !full_range {
return scalar::gray8_to_hsv_row(y_plane, h_out, s_out, v_out, width, full_range);
}
let mut x = 0usize;
unsafe {
let zero = _mm_setzero_si128();
while x + 16 <= width {
let v = _mm_loadu_si128(y_plane.as_ptr().add(x).cast());
_mm_storeu_si128(h_out.as_mut_ptr().add(x).cast(), zero);
_mm_storeu_si128(s_out.as_mut_ptr().add(x).cast(), zero);
_mm_storeu_si128(v_out.as_mut_ptr().add(x).cast(), v);
x += 16;
}
}
if x < width {
scalar::gray8_to_hsv_row(
&y_plane[x..width],
&mut h_out[x..width],
&mut s_out[x..width],
&mut v_out[x..width],
width - x,
true,
);
}
}
#[allow(dead_code)]
#[inline]
#[target_feature(enable = "sse4.1")]
pub(crate) unsafe fn gray_n_to_rgb_row<const BITS: u32, const BE: bool>(
y_plane: &[u16],
out: &mut [u8],
width: usize,
full_range: bool,
) {
debug_assert!(y_plane.len() >= width);
debug_assert!(out.len() >= width * 3);
scalar::gray_n_to_rgb_row::<BITS, BE>(y_plane, out, width, full_range);
}
#[allow(dead_code)]
#[inline]
#[target_feature(enable = "sse4.1")]
pub(crate) unsafe fn gray_n_to_rgba_row<const BITS: u32, const BE: bool>(
y_plane: &[u16],
out: &mut [u8],
width: usize,
full_range: bool,
) {
debug_assert!(y_plane.len() >= width);
debug_assert!(out.len() >= width * 4);
scalar::gray_n_to_rgba_row::<BITS, BE>(y_plane, out, width, full_range);
}
#[allow(dead_code)]
#[inline]
#[target_feature(enable = "sse4.1")]
pub(crate) unsafe fn gray_n_to_rgb_u16_row<const BITS: u32, const BE: bool>(
y_plane: &[u16],
out: &mut [u16],
width: usize,
full_range: bool,
) {
debug_assert!(y_plane.len() >= width);
debug_assert!(out.len() >= width * 3);
scalar::gray_n_to_rgb_u16_row::<BITS, BE>(y_plane, out, width, full_range);
}
#[allow(dead_code)]
#[inline]
#[target_feature(enable = "sse4.1")]
pub(crate) unsafe fn gray_n_to_rgba_u16_row<const BITS: u32, const BE: bool>(
y_plane: &[u16],
out: &mut [u16],
width: usize,
full_range: bool,
) {
debug_assert!(y_plane.len() >= width);
debug_assert!(out.len() >= width * 4);
scalar::gray_n_to_rgba_u16_row::<BITS, BE>(y_plane, out, width, full_range);
}
#[allow(dead_code)]
#[inline]
#[target_feature(enable = "sse4.1")]
pub(crate) unsafe fn gray_n_to_luma_row<const BITS: u32, const BE: bool>(
y_plane: &[u16],
out: &mut [u8],
width: usize,
) {
debug_assert!(y_plane.len() >= width);
debug_assert!(out.len() >= width);
let mask = bits_mask::<BITS>();
let mut x = 0usize;
unsafe {
let mask_v = _mm_set1_epi16(mask as i16);
let shr = _mm_cvtsi32_si128((BITS - 8) as i32);
while x + 8 <= width {
let raw = load_endian_u16x8::<BE>(y_plane.as_ptr().cast::<u8>().add(x * 2));
let masked = _mm_and_si128(raw, mask_v);
let shifted = _mm_srl_epi16(masked, shr);
let zero = _mm_setzero_si128();
let packed = _mm_packus_epi16(shifted, zero); let val = _mm_cvtsi128_si64(packed) as u64;
let bytes = val.to_le_bytes();
out[x..x + 8].copy_from_slice(&bytes);
x += 8;
}
}
if x < width {
scalar::gray_n_to_luma_row::<BITS, BE>(&y_plane[x..width], &mut out[x..width], width - x);
}
}
#[allow(dead_code)]
#[inline]
#[target_feature(enable = "sse4.1")]
pub(crate) unsafe fn gray_n_to_luma_u16_row<const BITS: u32, const BE: bool>(
y_plane: &[u16],
out: &mut [u16],
width: usize,
) {
debug_assert!(y_plane.len() >= width);
debug_assert!(out.len() >= width);
let mask = bits_mask::<BITS>();
let mut x = 0usize;
unsafe {
let mask_v = _mm_set1_epi16(mask as i16);
while x + 8 <= width {
let raw = load_endian_u16x8::<BE>(y_plane.as_ptr().cast::<u8>().add(x * 2));
let masked = _mm_and_si128(raw, mask_v);
_mm_storeu_si128(out.as_mut_ptr().add(x).cast(), masked);
x += 8;
}
}
if x < width {
scalar::gray_n_to_luma_u16_row::<BITS, BE>(&y_plane[x..width], &mut out[x..width], width - x);
}
}
#[allow(dead_code)]
#[inline]
#[target_feature(enable = "sse4.1")]
pub(crate) unsafe fn gray_n_to_hsv_row<const BITS: u32, const BE: bool>(
y_plane: &[u16],
h_out: &mut [u8],
s_out: &mut [u8],
v_out: &mut [u8],
width: usize,
full_range: bool,
) {
debug_assert!(y_plane.len() >= width);
if !full_range {
return scalar::gray_n_to_hsv_row::<BITS, BE>(y_plane, h_out, s_out, v_out, width, full_range);
}
let mask = bits_mask::<BITS>();
let mut x = 0usize;
unsafe {
let mask_v = _mm_set1_epi16(mask as i16);
let shr = _mm_cvtsi32_si128((BITS - 8) as i32);
let zero = _mm_setzero_si128();
while x + 8 <= width {
let raw = load_endian_u16x8::<BE>(y_plane.as_ptr().cast::<u8>().add(x * 2));
let masked = _mm_and_si128(raw, mask_v);
let shifted = _mm_srl_epi16(masked, shr);
let packed = _mm_packus_epi16(shifted, zero);
let val = _mm_cvtsi128_si64(packed) as u64;
let bytes = val.to_le_bytes();
h_out[x..x + 8].fill(0);
s_out[x..x + 8].fill(0);
v_out[x..x + 8].copy_from_slice(&bytes);
x += 8;
}
}
if x < width {
scalar::gray_n_to_hsv_row::<BITS, BE>(
&y_plane[x..width],
&mut h_out[x..width],
&mut s_out[x..width],
&mut v_out[x..width],
width - x,
true,
);
}
}
#[allow(dead_code)]
#[inline]
#[target_feature(enable = "sse4.1")]
pub(crate) unsafe fn gray16_to_rgb_row<const BE: bool>(
y_plane: &[u16],
out: &mut [u8],
width: usize,
full_range: bool,
) {
debug_assert!(y_plane.len() >= width);
debug_assert!(out.len() >= width * 3);
scalar::gray16_to_rgb_row::<BE>(y_plane, out, width, full_range);
}
#[allow(dead_code)]
#[inline]
#[target_feature(enable = "sse4.1")]
pub(crate) unsafe fn gray16_to_rgba_row<const BE: bool>(
y_plane: &[u16],
out: &mut [u8],
width: usize,
full_range: bool,
) {
debug_assert!(y_plane.len() >= width);
debug_assert!(out.len() >= width * 4);
scalar::gray16_to_rgba_row::<BE>(y_plane, out, width, full_range);
}
#[allow(dead_code)]
#[inline]
#[target_feature(enable = "sse4.1")]
pub(crate) unsafe fn gray16_to_rgb_u16_row<const BE: bool>(
y_plane: &[u16],
out: &mut [u16],
width: usize,
full_range: bool,
) {
debug_assert!(y_plane.len() >= width);
debug_assert!(out.len() >= width * 3);
scalar::gray16_to_rgb_u16_row::<BE>(y_plane, out, width, full_range);
}
#[allow(dead_code)]
#[inline]
#[target_feature(enable = "sse4.1")]
pub(crate) unsafe fn gray16_to_rgba_u16_row<const BE: bool>(
y_plane: &[u16],
out: &mut [u16],
width: usize,
full_range: bool,
) {
debug_assert!(y_plane.len() >= width);
debug_assert!(out.len() >= width * 4);
scalar::gray16_to_rgba_u16_row::<BE>(y_plane, out, width, full_range);
}
#[allow(dead_code)]
#[inline]
#[target_feature(enable = "sse4.1")]
pub(crate) unsafe fn gray16_to_luma_row<const BE: bool>(
y_plane: &[u16],
out: &mut [u8],
width: usize,
) {
debug_assert!(y_plane.len() >= width);
debug_assert!(out.len() >= width);
let mut x = 0usize;
unsafe {
let zero = _mm_setzero_si128();
while x + 8 <= width {
let raw = load_endian_u16x8::<BE>(y_plane.as_ptr().cast::<u8>().add(x * 2));
let shifted = _mm_srli_epi16(raw, 8);
let packed = _mm_packus_epi16(shifted, zero);
let val = _mm_cvtsi128_si64(packed) as u64;
out[x..x + 8].copy_from_slice(&val.to_le_bytes());
x += 8;
}
}
if x < width {
scalar::gray16_to_luma_row::<BE>(&y_plane[x..width], &mut out[x..width], width - x);
}
}
#[allow(dead_code)]
#[inline]
#[target_feature(enable = "sse4.1")]
pub(crate) unsafe fn gray16_to_luma_u16_row<const BE: bool>(
y_plane: &[u16],
out: &mut [u16],
width: usize,
) {
debug_assert!(y_plane.len() >= width);
debug_assert!(out.len() >= width);
let mut x = 0usize;
unsafe {
while x + 8 <= width {
let y = load_endian_u16x8::<BE>(y_plane.as_ptr().cast::<u8>().add(x * 2));
_mm_storeu_si128(out.as_mut_ptr().add(x).cast(), y);
x += 8;
}
}
if x < width {
scalar::gray16_to_luma_u16_row::<BE>(&y_plane[x..width], &mut out[x..width], width - x);
}
}
#[allow(dead_code)]
#[inline]
#[target_feature(enable = "sse4.1")]
pub(crate) unsafe fn gray16_to_hsv_row<const BE: bool>(
y_plane: &[u16],
h_out: &mut [u8],
s_out: &mut [u8],
v_out: &mut [u8],
width: usize,
full_range: bool,
) {
debug_assert!(y_plane.len() >= width);
if !full_range {
return scalar::gray16_to_hsv_row::<BE>(y_plane, h_out, s_out, v_out, width, full_range);
}
let mut x = 0usize;
unsafe {
let zero16 = _mm_setzero_si128();
while x + 8 <= width {
let raw = load_endian_u16x8::<BE>(y_plane.as_ptr().cast::<u8>().add(x * 2));
let shifted = _mm_srli_epi16(raw, 8);
let packed = _mm_packus_epi16(shifted, zero16);
let val = _mm_cvtsi128_si64(packed) as u64;
h_out[x..x + 8].fill(0);
s_out[x..x + 8].fill(0);
v_out[x..x + 8].copy_from_slice(&val.to_le_bytes());
x += 8;
}
}
if x < width {
scalar::gray16_to_hsv_row::<BE>(
&y_plane[x..width],
&mut h_out[x..width],
&mut s_out[x..width],
&mut v_out[x..width],
width - x,
true,
);
}
}
#[allow(dead_code)]
#[inline]
#[target_feature(enable = "sse4.1")]
pub(crate) unsafe fn gray32_to_rgb_row<const BE: bool>(
y_plane: &[u32],
out: &mut [u8],
width: usize,
full_range: bool,
) {
debug_assert!(y_plane.len() >= width);
debug_assert!(out.len() >= width * 3);
scalar::gray32_to_rgb_row::<BE>(y_plane, out, width, full_range);
}
#[allow(dead_code)]
#[inline]
#[target_feature(enable = "sse4.1")]
pub(crate) unsafe fn gray32_to_rgba_row<const BE: bool>(
y_plane: &[u32],
out: &mut [u8],
width: usize,
full_range: bool,
) {
debug_assert!(y_plane.len() >= width);
debug_assert!(out.len() >= width * 4);
scalar::gray32_to_rgba_row::<BE>(y_plane, out, width, full_range);
}
#[allow(dead_code)]
#[inline]
#[target_feature(enable = "sse4.1")]
pub(crate) unsafe fn gray32_to_rgb_u16_row<const BE: bool>(
y_plane: &[u32],
out: &mut [u16],
width: usize,
full_range: bool,
) {
debug_assert!(y_plane.len() >= width);
debug_assert!(out.len() >= width * 3);
scalar::gray32_to_rgb_u16_row::<BE>(y_plane, out, width, full_range);
}
#[allow(dead_code)]
#[inline]
#[target_feature(enable = "sse4.1")]
pub(crate) unsafe fn gray32_to_rgba_u16_row<const BE: bool>(
y_plane: &[u32],
out: &mut [u16],
width: usize,
full_range: bool,
) {
debug_assert!(y_plane.len() >= width);
debug_assert!(out.len() >= width * 4);
scalar::gray32_to_rgba_u16_row::<BE>(y_plane, out, width, full_range);
}
#[allow(dead_code)]
#[inline]
#[target_feature(enable = "sse4.1")]
pub(crate) unsafe fn gray32_to_luma_row<const BE: bool>(
y_plane: &[u32],
out: &mut [u8],
width: usize,
) {
debug_assert!(y_plane.len() >= width);
debug_assert!(out.len() >= width);
let mut x = 0usize;
unsafe {
let zero = _mm_setzero_si128();
while x + 8 <= width {
let lo = load_endian_u32x4::<BE>(y_plane.as_ptr().cast::<u8>().add(x * 4));
let hi = load_endian_u32x4::<BE>(y_plane.as_ptr().cast::<u8>().add((x + 4) * 4));
let lo_s = _mm_srli_epi32(lo, 24);
let hi_s = _mm_srli_epi32(hi, 24);
let packed16 = _mm_packus_epi32(lo_s, hi_s);
let packed8 = _mm_packus_epi16(packed16, zero);
let val = _mm_cvtsi128_si64(packed8) as u64;
out[x..x + 8].copy_from_slice(&val.to_le_bytes());
x += 8;
}
}
if x < width {
scalar::gray32_to_luma_row::<BE>(&y_plane[x..width], &mut out[x..width], width - x);
}
}
#[allow(dead_code)]
#[inline]
#[target_feature(enable = "sse4.1")]
pub(crate) unsafe fn gray32_to_luma_u16_row<const BE: bool>(
y_plane: &[u32],
out: &mut [u16],
width: usize,
) {
debug_assert!(y_plane.len() >= width);
debug_assert!(out.len() >= width);
let mut x = 0usize;
unsafe {
while x + 8 <= width {
let lo = load_endian_u32x4::<BE>(y_plane.as_ptr().cast::<u8>().add(x * 4));
let hi = load_endian_u32x4::<BE>(y_plane.as_ptr().cast::<u8>().add((x + 4) * 4));
let lo_s = _mm_srli_epi32(lo, 16);
let hi_s = _mm_srli_epi32(hi, 16);
let packed = _mm_packus_epi32(lo_s, hi_s);
_mm_storeu_si128(out.as_mut_ptr().add(x).cast(), packed);
x += 8;
}
}
if x < width {
scalar::gray32_to_luma_u16_row::<BE>(&y_plane[x..width], &mut out[x..width], width - x);
}
}
#[allow(dead_code)]
#[inline]
#[target_feature(enable = "sse4.1")]
pub(crate) unsafe fn gray32_to_hsv_row<const BE: bool>(
y_plane: &[u32],
h_out: &mut [u8],
s_out: &mut [u8],
v_out: &mut [u8],
width: usize,
full_range: bool,
) {
debug_assert!(y_plane.len() >= width);
if !full_range {
return scalar::gray32_to_hsv_row::<BE>(y_plane, h_out, s_out, v_out, width, full_range);
}
let mut x = 0usize;
unsafe {
let zero = _mm_setzero_si128();
while x + 8 <= width {
let lo = load_endian_u32x4::<BE>(y_plane.as_ptr().cast::<u8>().add(x * 4));
let hi = load_endian_u32x4::<BE>(y_plane.as_ptr().cast::<u8>().add((x + 4) * 4));
let lo_s = _mm_srli_epi32(lo, 24);
let hi_s = _mm_srli_epi32(hi, 24);
let packed16 = _mm_packus_epi32(lo_s, hi_s);
let packed8 = _mm_packus_epi16(packed16, zero);
let val = _mm_cvtsi128_si64(packed8) as u64;
h_out[x..x + 8].fill(0);
s_out[x..x + 8].fill(0);
v_out[x..x + 8].copy_from_slice(&val.to_le_bytes());
x += 8;
}
}
if x < width {
scalar::gray32_to_hsv_row::<BE>(
&y_plane[x..width],
&mut h_out[x..width],
&mut s_out[x..width],
&mut v_out[x..width],
width - x,
true,
);
}
}
#[inline]
#[target_feature(enable = "sse4.1")]
pub(crate) unsafe fn grayf32_to_rgb_row<const BE: bool>(
y_plane: &[f32],
out: &mut [u8],
width: usize,
) {
use crate::row::scalar::grayf32 as scalar;
debug_assert!(y_plane.len() >= width);
debug_assert!(out.len() >= width * 3);
let scale = _mm_set1_ps(255.0);
let zero = _mm_setzero_ps();
let one = _mm_set1_ps(1.0);
let mut x = 0usize;
unsafe {
while x + 4 <= width {
let y = _mm_castsi128_ps(load_endian_u32x4::<BE>(
y_plane.as_ptr().cast::<u8>().add(x * 4),
));
let clamped = _mm_min_ps(_mm_max_ps(y, zero), one);
let scaled = _mm_mul_ps(clamped, scale);
let int32 = _mm_cvttps_epi32(_mm_add_ps(scaled, _mm_set1_ps(0.5)));
let pack16 = _mm_packs_epi32(int32, int32);
let pack8 = _mm_packus_epi16(pack16, pack16);
let v0 = _mm_extract_epi8(pack8, 0) as u8;
let v1 = _mm_extract_epi8(pack8, 1) as u8;
let v2 = _mm_extract_epi8(pack8, 2) as u8;
let v3 = _mm_extract_epi8(pack8, 3) as u8;
let base = x * 3;
out[base] = v0;
out[base + 1] = v0;
out[base + 2] = v0;
out[base + 3] = v1;
out[base + 4] = v1;
out[base + 5] = v1;
out[base + 6] = v2;
out[base + 7] = v2;
out[base + 8] = v2;
out[base + 9] = v3;
out[base + 10] = v3;
out[base + 11] = v3;
x += 4;
}
}
if x < width {
scalar::grayf32_to_rgb_row::<BE>(&y_plane[x..width], &mut out[x * 3..width * 3], width - x);
}
}
#[inline]
#[target_feature(enable = "sse4.1")]
pub(crate) unsafe fn grayf32_to_rgba_row<const BE: bool>(
y_plane: &[f32],
out: &mut [u8],
width: usize,
) {
use crate::row::scalar::grayf32 as scalar;
debug_assert!(y_plane.len() >= width);
debug_assert!(out.len() >= width * 4);
let scale = _mm_set1_ps(255.0);
let zero = _mm_setzero_ps();
let one = _mm_set1_ps(1.0);
let mut x = 0usize;
unsafe {
while x + 4 <= width {
let y = _mm_castsi128_ps(load_endian_u32x4::<BE>(
y_plane.as_ptr().cast::<u8>().add(x * 4),
));
let clamped = _mm_min_ps(_mm_max_ps(y, zero), one);
let scaled = _mm_mul_ps(clamped, scale);
let int32 = _mm_cvttps_epi32(_mm_add_ps(scaled, _mm_set1_ps(0.5)));
let pack16 = _mm_packs_epi32(int32, int32);
let pack8 = _mm_packus_epi16(pack16, pack16);
let v0 = _mm_extract_epi8(pack8, 0) as u8;
let v1 = _mm_extract_epi8(pack8, 1) as u8;
let v2 = _mm_extract_epi8(pack8, 2) as u8;
let v3 = _mm_extract_epi8(pack8, 3) as u8;
let base = x * 4;
out[base] = v0;
out[base + 1] = v0;
out[base + 2] = v0;
out[base + 3] = 0xFF;
out[base + 4] = v1;
out[base + 5] = v1;
out[base + 6] = v1;
out[base + 7] = 0xFF;
out[base + 8] = v2;
out[base + 9] = v2;
out[base + 10] = v2;
out[base + 11] = 0xFF;
out[base + 12] = v3;
out[base + 13] = v3;
out[base + 14] = v3;
out[base + 15] = 0xFF;
x += 4;
}
}
if x < width {
scalar::grayf32_to_rgba_row::<BE>(&y_plane[x..width], &mut out[x * 4..width * 4], width - x);
}
}
#[inline]
#[target_feature(enable = "sse4.1")]
pub(crate) unsafe fn grayf32_to_rgb_u16_row<const BE: bool>(
y_plane: &[f32],
out: &mut [u16],
width: usize,
) {
use crate::row::scalar::grayf32 as scalar;
debug_assert!(y_plane.len() >= width);
debug_assert!(out.len() >= width * 3);
let scale = _mm_set1_ps(65535.0);
let zero = _mm_setzero_ps();
let one = _mm_set1_ps(1.0);
let mut x = 0usize;
unsafe {
while x + 4 <= width {
let y = _mm_castsi128_ps(load_endian_u32x4::<BE>(
y_plane.as_ptr().cast::<u8>().add(x * 4),
));
let clamped = _mm_min_ps(_mm_max_ps(y, zero), one);
let scaled = _mm_mul_ps(clamped, scale);
let int32 = _mm_cvttps_epi32(_mm_add_ps(scaled, _mm_set1_ps(0.5)));
let base = x * 3;
let v0 = _mm_extract_epi32::<0>(int32) as u16;
let v1 = _mm_extract_epi32::<1>(int32) as u16;
let v2 = _mm_extract_epi32::<2>(int32) as u16;
let v3 = _mm_extract_epi32::<3>(int32) as u16;
out[base] = v0;
out[base + 1] = v0;
out[base + 2] = v0;
out[base + 3] = v1;
out[base + 4] = v1;
out[base + 5] = v1;
out[base + 6] = v2;
out[base + 7] = v2;
out[base + 8] = v2;
out[base + 9] = v3;
out[base + 10] = v3;
out[base + 11] = v3;
x += 4;
}
}
if x < width {
scalar::grayf32_to_rgb_u16_row::<BE>(&y_plane[x..width], &mut out[x * 3..width * 3], width - x);
}
}
#[inline]
#[target_feature(enable = "sse4.1")]
pub(crate) unsafe fn grayf32_to_rgba_u16_row<const BE: bool>(
y_plane: &[f32],
out: &mut [u16],
width: usize,
) {
use crate::row::scalar::grayf32 as scalar;
debug_assert!(y_plane.len() >= width);
debug_assert!(out.len() >= width * 4);
let scale = _mm_set1_ps(65535.0);
let zero = _mm_setzero_ps();
let one = _mm_set1_ps(1.0);
let mut x = 0usize;
unsafe {
while x + 4 <= width {
let y = _mm_castsi128_ps(load_endian_u32x4::<BE>(
y_plane.as_ptr().cast::<u8>().add(x * 4),
));
let clamped = _mm_min_ps(_mm_max_ps(y, zero), one);
let scaled = _mm_mul_ps(clamped, scale);
let int32 = _mm_cvttps_epi32(_mm_add_ps(scaled, _mm_set1_ps(0.5)));
let base = x * 4;
let v0 = _mm_extract_epi32::<0>(int32) as u16;
let v1 = _mm_extract_epi32::<1>(int32) as u16;
let v2 = _mm_extract_epi32::<2>(int32) as u16;
let v3 = _mm_extract_epi32::<3>(int32) as u16;
out[base] = v0;
out[base + 1] = v0;
out[base + 2] = v0;
out[base + 3] = 0xFFFF;
out[base + 4] = v1;
out[base + 5] = v1;
out[base + 6] = v1;
out[base + 7] = 0xFFFF;
out[base + 8] = v2;
out[base + 9] = v2;
out[base + 10] = v2;
out[base + 11] = 0xFFFF;
out[base + 12] = v3;
out[base + 13] = v3;
out[base + 14] = v3;
out[base + 15] = 0xFFFF;
x += 4;
}
}
if x < width {
scalar::grayf32_to_rgba_u16_row::<BE>(
&y_plane[x..width],
&mut out[x * 4..width * 4],
width - x,
);
}
}
#[allow(dead_code)] #[inline]
#[target_feature(enable = "sse4.1")]
pub(crate) unsafe fn grayf32_to_rgb_f32_row<const BE: bool>(
y_plane: &[f32],
out: &mut [f32],
width: usize,
) {
use crate::row::scalar::grayf32 as scalar;
debug_assert!(y_plane.len() >= width);
debug_assert!(out.len() >= width * 3);
scalar::grayf32_to_rgb_f32_row::<BE>(y_plane, out, width);
}
#[inline]
#[target_feature(enable = "sse4.1")]
pub(crate) unsafe fn grayf32_to_luma_row<const BE: bool>(
y_plane: &[f32],
out: &mut [u8],
width: usize,
) {
use crate::row::scalar::grayf32 as scalar;
debug_assert!(y_plane.len() >= width);
debug_assert!(out.len() >= width);
let scale = _mm_set1_ps(255.0);
let zero = _mm_setzero_ps();
let one = _mm_set1_ps(1.0);
let mut x = 0usize;
unsafe {
while x + 4 <= width {
let y = _mm_castsi128_ps(load_endian_u32x4::<BE>(
y_plane.as_ptr().cast::<u8>().add(x * 4),
));
let clamped = _mm_min_ps(_mm_max_ps(y, zero), one);
let scaled = _mm_mul_ps(clamped, scale);
let int32 = _mm_cvttps_epi32(_mm_add_ps(scaled, _mm_set1_ps(0.5)));
let pack16 = _mm_packs_epi32(int32, int32);
let pack8 = _mm_packus_epi16(pack16, pack16);
let val = _mm_cvtsi128_si32(pack8) as u32;
out[x..x + 4].copy_from_slice(&val.to_le_bytes());
x += 4;
}
}
if x < width {
scalar::grayf32_to_luma_row::<BE>(&y_plane[x..width], &mut out[x..width], width - x);
}
}
#[inline]
#[target_feature(enable = "sse4.1")]
pub(crate) unsafe fn grayf32_to_luma_u16_row<const BE: bool>(
y_plane: &[f32],
out: &mut [u16],
width: usize,
) {
use crate::row::scalar::grayf32 as scalar;
debug_assert!(y_plane.len() >= width);
debug_assert!(out.len() >= width);
let scale = _mm_set1_ps(65535.0);
let zero = _mm_setzero_ps();
let one = _mm_set1_ps(1.0);
let mut x = 0usize;
unsafe {
while x + 4 <= width {
let y = _mm_castsi128_ps(load_endian_u32x4::<BE>(
y_plane.as_ptr().cast::<u8>().add(x * 4),
));
let clamped = _mm_min_ps(_mm_max_ps(y, zero), one);
let scaled = _mm_mul_ps(clamped, scale);
let int32 = _mm_cvttps_epi32(_mm_add_ps(scaled, _mm_set1_ps(0.5)));
let pack16 = _mm_packus_epi32(int32, int32);
_mm_storel_epi64(out.as_mut_ptr().add(x).cast::<__m128i>(), pack16);
x += 4;
}
}
if x < width {
scalar::grayf32_to_luma_u16_row::<BE>(&y_plane[x..width], &mut out[x..width], width - x);
}
}
#[allow(dead_code)] #[inline]
#[target_feature(enable = "sse4.1")]
pub(crate) unsafe fn grayf32_to_luma_f32_row<const BE: bool>(
y_plane: &[f32],
out: &mut [f32],
width: usize,
) {
use crate::row::scalar::grayf32 as scalar;
debug_assert!(y_plane.len() >= width);
debug_assert!(out.len() >= width);
scalar::grayf32_to_luma_f32_row::<BE>(y_plane, out, width);
}
#[inline]
#[target_feature(enable = "sse4.1")]
pub(crate) unsafe fn grayf32_to_hsv_row<const BE: bool>(
y_plane: &[f32],
h_out: &mut [u8],
s_out: &mut [u8],
v_out: &mut [u8],
width: usize,
) {
use crate::row::scalar::grayf32 as scalar;
debug_assert!(y_plane.len() >= width);
let scale = _mm_set1_ps(255.0);
let zero = _mm_setzero_ps();
let one = _mm_set1_ps(1.0);
let zero128 = _mm_setzero_si128();
let mut x = 0usize;
unsafe {
while x + 4 <= width {
let y = _mm_castsi128_ps(load_endian_u32x4::<BE>(
y_plane.as_ptr().cast::<u8>().add(x * 4),
));
let clamped = _mm_min_ps(_mm_max_ps(y, zero), one);
let scaled = _mm_mul_ps(clamped, scale);
let int32 = _mm_cvttps_epi32(_mm_add_ps(scaled, _mm_set1_ps(0.5)));
let pack16 = _mm_packs_epi32(int32, int32);
let pack8 = _mm_packus_epi16(pack16, pack16);
let val = _mm_cvtsi128_si32(pack8) as u32;
let vbytes = val.to_le_bytes();
h_out[x..x + 4].copy_from_slice(&[0u8; 4]);
s_out[x..x + 4].copy_from_slice(&[0u8; 4]);
v_out[x..x + 4].copy_from_slice(&vbytes);
let _ = zero128;
x += 4;
}
}
if x < width {
scalar::grayf32_to_hsv_row::<BE>(
&y_plane[x..width],
&mut h_out[x..width],
&mut s_out[x..width],
&mut v_out[x..width],
width - x,
);
}
}
#[inline]
#[target_feature(enable = "sse4.1")]
pub(crate) unsafe fn ya8_to_rgb_row(packed: &[u8], out: &mut [u8], width: usize) {
use crate::row::scalar::ya8 as scalar;
debug_assert!(packed.len() >= width * 2);
debug_assert!(out.len() >= width * 3);
let mut x = 0usize;
unsafe {
let y_mask = _mm_set_epi8(
-128, -128, -128, -128, -128, -128, -128, -128, 14, 12, 10, 8, 6, 4, 2, 0,
);
while x + 8 <= width {
let chunk = _mm_loadu_si128(packed.as_ptr().add(x * 2).cast());
let y_bytes = _mm_shuffle_epi8(chunk, y_mask); let y_lo = _mm_cvtsi128_si64(y_bytes) as u64;
let ybuf = y_lo.to_le_bytes();
let base = x * 3;
for i in 0..8usize {
out[base + i * 3] = ybuf[i];
out[base + i * 3 + 1] = ybuf[i];
out[base + i * 3 + 2] = ybuf[i];
}
x += 8;
}
}
if x < width {
scalar::ya8_to_rgb_row(
&packed[x * 2..width * 2],
&mut out[x * 3..width * 3],
width - x,
);
}
}
#[inline]
#[target_feature(enable = "sse4.1")]
pub(crate) unsafe fn ya8_to_rgba_row(packed: &[u8], out: &mut [u8], width: usize) {
use crate::row::scalar::ya8 as scalar;
debug_assert!(packed.len() >= width * 2);
debug_assert!(out.len() >= width * 4);
let mut x = 0usize;
unsafe {
let y_mask = _mm_set_epi8(
-128, -128, -128, -128, -128, -128, -128, -128, 14, 12, 10, 8, 6, 4, 2, 0,
);
let a_mask = _mm_set_epi8(
-128, -128, -128, -128, -128, -128, -128, -128, 15, 13, 11, 9, 7, 5, 3, 1,
);
while x + 8 <= width {
let chunk = _mm_loadu_si128(packed.as_ptr().add(x * 2).cast());
let y_bytes = _mm_shuffle_epi8(chunk, y_mask);
let a_bytes = _mm_shuffle_epi8(chunk, a_mask);
let y_lo = _mm_cvtsi128_si64(y_bytes) as u64;
let a_lo = _mm_cvtsi128_si64(a_bytes) as u64;
let ybuf = y_lo.to_le_bytes();
let abuf = a_lo.to_le_bytes();
let base = x * 4;
for i in 0..8usize {
out[base + i * 4] = ybuf[i];
out[base + i * 4 + 1] = ybuf[i];
out[base + i * 4 + 2] = ybuf[i];
out[base + i * 4 + 3] = abuf[i];
}
x += 8;
}
}
if x < width {
scalar::ya8_to_rgba_row(
&packed[x * 2..width * 2],
&mut out[x * 4..width * 4],
width - x,
);
}
}
#[inline]
#[target_feature(enable = "sse4.1")]
pub(crate) unsafe fn ya8_to_rgb_u16_row(packed: &[u8], out: &mut [u16], width: usize) {
use crate::row::scalar::ya8 as scalar;
debug_assert!(packed.len() >= width * 2);
debug_assert!(out.len() >= width * 3);
scalar::ya8_to_rgb_u16_row(packed, out, width);
}
#[inline]
#[target_feature(enable = "sse4.1")]
pub(crate) unsafe fn ya8_to_rgba_u16_row(packed: &[u8], out: &mut [u16], width: usize) {
use crate::row::scalar::ya8 as scalar;
debug_assert!(packed.len() >= width * 2);
debug_assert!(out.len() >= width * 4);
scalar::ya8_to_rgba_u16_row(packed, out, width);
}
#[inline]
#[target_feature(enable = "sse4.1")]
pub(crate) unsafe fn ya8_to_luma_row(packed: &[u8], out: &mut [u8], width: usize) {
use crate::row::scalar::ya8 as scalar;
debug_assert!(packed.len() >= width * 2);
debug_assert!(out.len() >= width);
let mut x = 0usize;
unsafe {
let y_mask = _mm_set_epi8(
-128, -128, -128, -128, -128, -128, -128, -128, 14, 12, 10, 8, 6, 4, 2, 0,
);
while x + 8 <= width {
let chunk = _mm_loadu_si128(packed.as_ptr().add(x * 2).cast());
let y_bytes = _mm_shuffle_epi8(chunk, y_mask);
let val = _mm_cvtsi128_si64(y_bytes) as u64;
out[x..x + 8].copy_from_slice(&val.to_le_bytes());
x += 8;
}
}
if x < width {
scalar::ya8_to_luma_row(&packed[x * 2..width * 2], &mut out[x..width], width - x);
}
}
#[inline]
#[target_feature(enable = "sse4.1")]
pub(crate) unsafe fn ya8_to_luma_u16_row(packed: &[u8], out: &mut [u16], width: usize) {
use crate::row::scalar::ya8 as scalar;
debug_assert!(packed.len() >= width * 2);
debug_assert!(out.len() >= width);
scalar::ya8_to_luma_u16_row(packed, out, width);
}
#[inline]
#[target_feature(enable = "sse4.1")]
pub(crate) unsafe fn ya8_to_hsv_row(
packed: &[u8],
h_out: &mut [u8],
s_out: &mut [u8],
v_out: &mut [u8],
width: usize,
) {
use crate::row::scalar::ya8 as scalar;
debug_assert!(packed.len() >= width * 2);
let mut x = 0usize;
unsafe {
let y_mask = _mm_set_epi8(
-128, -128, -128, -128, -128, -128, -128, -128, 14, 12, 10, 8, 6, 4, 2, 0,
);
while x + 8 <= width {
let chunk = _mm_loadu_si128(packed.as_ptr().add(x * 2).cast());
let y_bytes = _mm_shuffle_epi8(chunk, y_mask);
let val = _mm_cvtsi128_si64(y_bytes) as u64;
let vbytes = val.to_le_bytes();
h_out[x..x + 8].fill(0);
s_out[x..x + 8].fill(0);
v_out[x..x + 8].copy_from_slice(&vbytes);
x += 8;
}
}
if x < width {
scalar::ya8_to_hsv_row(
&packed[x * 2..width * 2],
&mut h_out[x..width],
&mut s_out[x..width],
&mut v_out[x..width],
width - x,
);
}
}
const HOST_NATIVE_BE: bool = cfg!(target_endian = "big");
#[inline]
#[target_feature(enable = "sse4.1")]
pub(crate) unsafe fn ya16_to_rgb_row<const BE: bool>(packed: &[u16], out: &mut [u8], width: usize) {
use crate::row::scalar::ya16 as scalar;
debug_assert!(packed.len() >= width * 2);
debug_assert!(out.len() >= width * 3);
if BE != HOST_NATIVE_BE {
return scalar::ya16_to_rgb_row::<BE>(packed, out, width);
}
let mut x = 0usize;
unsafe {
let y_mask = _mm_set_epi8(
-128, -128, -128, -128, -128, -128, -128, -128, 13, 12, 9, 8, 5, 4, 1, 0,
);
while x + 4 <= width {
let chunk = _mm_loadu_si128(packed.as_ptr().add(x * 2).cast::<__m128i>());
let y_words = _mm_shuffle_epi8(chunk, y_mask); let y_shifted = _mm_srli_epi16(y_words, 8); let pack8 = _mm_packus_epi16(y_shifted, _mm_setzero_si128()); let val = _mm_cvtsi128_si32(pack8) as u32;
let ybuf = val.to_le_bytes();
let base = x * 3;
for i in 0..4usize {
out[base + i * 3] = ybuf[i];
out[base + i * 3 + 1] = ybuf[i];
out[base + i * 3 + 2] = ybuf[i];
}
x += 4;
}
}
if x < width {
scalar::ya16_to_rgb_row::<BE>(
&packed[x * 2..width * 2],
&mut out[x * 3..width * 3],
width - x,
);
}
}
#[inline]
#[target_feature(enable = "sse4.1")]
pub(crate) unsafe fn ya16_to_rgba_row<const BE: bool>(
packed: &[u16],
out: &mut [u8],
width: usize,
) {
use crate::row::scalar::ya16 as scalar;
debug_assert!(packed.len() >= width * 2);
debug_assert!(out.len() >= width * 4);
if BE != HOST_NATIVE_BE {
return scalar::ya16_to_rgba_row::<BE>(packed, out, width);
}
let mut x = 0usize;
unsafe {
let y_mask = _mm_set_epi8(
-128, -128, -128, -128, -128, -128, -128, -128, 13, 12, 9, 8, 5, 4, 1, 0,
);
let a_mask = _mm_set_epi8(
-128, -128, -128, -128, -128, -128, -128, -128, 15, 14, 11, 10, 7, 6, 3, 2,
);
while x + 4 <= width {
let chunk = _mm_loadu_si128(packed.as_ptr().add(x * 2).cast::<__m128i>());
let y_words = _mm_shuffle_epi8(chunk, y_mask);
let a_words = _mm_shuffle_epi8(chunk, a_mask);
let y_shifted = _mm_srli_epi16(y_words, 8);
let a_shifted = _mm_srli_epi16(a_words, 8);
let zero = _mm_setzero_si128();
let y8 = _mm_packus_epi16(y_shifted, zero);
let a8 = _mm_packus_epi16(a_shifted, zero);
let yval = _mm_cvtsi128_si32(y8) as u32;
let aval = _mm_cvtsi128_si32(a8) as u32;
let ybuf = yval.to_le_bytes();
let abuf = aval.to_le_bytes();
let base = x * 4;
for i in 0..4usize {
out[base + i * 4] = ybuf[i];
out[base + i * 4 + 1] = ybuf[i];
out[base + i * 4 + 2] = ybuf[i];
out[base + i * 4 + 3] = abuf[i];
}
x += 4;
}
}
if x < width {
scalar::ya16_to_rgba_row::<BE>(
&packed[x * 2..width * 2],
&mut out[x * 4..width * 4],
width - x,
);
}
}
#[inline]
#[target_feature(enable = "sse4.1")]
pub(crate) unsafe fn ya16_to_rgb_u16_row<const BE: bool>(
packed: &[u16],
out: &mut [u16],
width: usize,
) {
use crate::row::scalar::ya16 as scalar;
debug_assert!(packed.len() >= width * 2);
debug_assert!(out.len() >= width * 3);
scalar::ya16_to_rgb_u16_row::<BE>(packed, out, width);
}
#[inline]
#[target_feature(enable = "sse4.1")]
pub(crate) unsafe fn ya16_to_rgba_u16_row<const BE: bool>(
packed: &[u16],
out: &mut [u16],
width: usize,
) {
use crate::row::scalar::ya16 as scalar;
debug_assert!(packed.len() >= width * 2);
debug_assert!(out.len() >= width * 4);
scalar::ya16_to_rgba_u16_row::<BE>(packed, out, width);
}
#[inline]
#[target_feature(enable = "sse4.1")]
pub(crate) unsafe fn ya16_to_luma_row<const BE: bool>(
packed: &[u16],
out: &mut [u8],
width: usize,
) {
use crate::row::scalar::ya16 as scalar;
debug_assert!(packed.len() >= width * 2);
debug_assert!(out.len() >= width);
if BE != HOST_NATIVE_BE {
return scalar::ya16_to_luma_row::<BE>(packed, out, width);
}
let mut x = 0usize;
unsafe {
let y_mask = _mm_set_epi8(
-128, -128, -128, -128, -128, -128, -128, -128, 13, 12, 9, 8, 5, 4, 1, 0,
);
while x + 4 <= width {
let chunk = _mm_loadu_si128(packed.as_ptr().add(x * 2).cast::<__m128i>());
let y_words = _mm_shuffle_epi8(chunk, y_mask);
let y_shifted = _mm_srli_epi16(y_words, 8);
let pack8 = _mm_packus_epi16(y_shifted, _mm_setzero_si128());
let val = _mm_cvtsi128_si32(pack8) as u32;
out[x..x + 4].copy_from_slice(&val.to_le_bytes());
x += 4;
}
}
if x < width {
scalar::ya16_to_luma_row::<BE>(&packed[x * 2..width * 2], &mut out[x..width], width - x);
}
}
#[inline]
#[target_feature(enable = "sse4.1")]
pub(crate) unsafe fn ya16_to_luma_u16_row<const BE: bool>(
packed: &[u16],
out: &mut [u16],
width: usize,
) {
use crate::row::scalar::ya16 as scalar;
debug_assert!(packed.len() >= width * 2);
debug_assert!(out.len() >= width);
scalar::ya16_to_luma_u16_row::<BE>(packed, out, width);
}
#[inline]
#[target_feature(enable = "sse4.1")]
pub(crate) unsafe fn ya16_to_hsv_row<const BE: bool>(
packed: &[u16],
h_out: &mut [u8],
s_out: &mut [u8],
v_out: &mut [u8],
width: usize,
) {
use crate::row::scalar::ya16 as scalar;
debug_assert!(packed.len() >= width * 2);
if BE != HOST_NATIVE_BE {
return scalar::ya16_to_hsv_row::<BE>(packed, h_out, s_out, v_out, width);
}
let mut x = 0usize;
unsafe {
let y_mask = _mm_set_epi8(
-128, -128, -128, -128, -128, -128, -128, -128, 13, 12, 9, 8, 5, 4, 1, 0,
);
while x + 4 <= width {
let chunk = _mm_loadu_si128(packed.as_ptr().add(x * 2).cast::<__m128i>());
let y_words = _mm_shuffle_epi8(chunk, y_mask);
let y_shifted = _mm_srli_epi16(y_words, 8);
let pack8 = _mm_packus_epi16(y_shifted, _mm_setzero_si128());
let val = _mm_cvtsi128_si32(pack8) as u32;
let vbytes = val.to_le_bytes();
h_out[x..x + 4].fill(0);
s_out[x..x + 4].fill(0);
v_out[x..x + 4].copy_from_slice(&vbytes);
x += 4;
}
}
if x < width {
scalar::ya16_to_hsv_row::<BE>(
&packed[x * 2..width * 2],
&mut h_out[x..width],
&mut s_out[x..width],
&mut v_out[x..width],
width - x,
);
}
}
const GRAYF16_HOST_NATIVE_BE: bool = cfg!(target_endian = "big");
#[inline]
#[target_feature(enable = "sse4.1,f16c")]
unsafe fn widen_f16x4_sse_buf<const BE: bool>(ptr: *const half::f16, out: *mut f32) {
unsafe {
let m = _mm_cvtph_ps(load_endian_u16x4::<BE>(ptr.cast::<u8>()));
_mm_storeu_ps(out, m);
}
}
#[inline]
#[target_feature(enable = "sse4.1,f16c")]
pub(crate) unsafe fn grayf16_to_rgb_row<const BE: bool>(
y_plane: &[half::f16],
out: &mut [u8],
width: usize,
) {
use crate::row::scalar::grayf16 as scalar;
debug_assert!(y_plane.len() >= width);
debug_assert!(out.len() >= width * 3);
let mut x = 0usize;
while x + 4 <= width {
let mut buf = [0.0f32; 4];
unsafe {
widen_f16x4_sse_buf::<BE>(y_plane.as_ptr().add(x), buf.as_mut_ptr());
grayf32_to_rgb_row::<GRAYF16_HOST_NATIVE_BE>(&buf, &mut out[x * 3..(x + 4) * 3], 4);
}
x += 4;
}
if x < width {
scalar::grayf16_to_rgb_row::<BE>(&y_plane[x..width], &mut out[x * 3..width * 3], width - x);
}
}
#[inline]
#[target_feature(enable = "sse4.1,f16c")]
pub(crate) unsafe fn grayf16_to_rgba_row<const BE: bool>(
y_plane: &[half::f16],
out: &mut [u8],
width: usize,
) {
use crate::row::scalar::grayf16 as scalar;
debug_assert!(y_plane.len() >= width);
debug_assert!(out.len() >= width * 4);
let mut x = 0usize;
while x + 4 <= width {
let mut buf = [0.0f32; 4];
unsafe {
widen_f16x4_sse_buf::<BE>(y_plane.as_ptr().add(x), buf.as_mut_ptr());
grayf32_to_rgba_row::<GRAYF16_HOST_NATIVE_BE>(&buf, &mut out[x * 4..(x + 4) * 4], 4);
}
x += 4;
}
if x < width {
scalar::grayf16_to_rgba_row::<BE>(&y_plane[x..width], &mut out[x * 4..width * 4], width - x);
}
}
#[inline]
#[target_feature(enable = "sse4.1,f16c")]
pub(crate) unsafe fn grayf16_to_rgb_u16_row<const BE: bool>(
y_plane: &[half::f16],
out: &mut [u16],
width: usize,
) {
use crate::row::scalar::grayf16 as scalar;
debug_assert!(y_plane.len() >= width);
debug_assert!(out.len() >= width * 3);
let mut x = 0usize;
while x + 4 <= width {
let mut buf = [0.0f32; 4];
unsafe {
widen_f16x4_sse_buf::<BE>(y_plane.as_ptr().add(x), buf.as_mut_ptr());
grayf32_to_rgb_u16_row::<GRAYF16_HOST_NATIVE_BE>(&buf, &mut out[x * 3..(x + 4) * 3], 4);
}
x += 4;
}
if x < width {
scalar::grayf16_to_rgb_u16_row::<BE>(&y_plane[x..width], &mut out[x * 3..width * 3], width - x);
}
}
#[inline]
#[target_feature(enable = "sse4.1,f16c")]
pub(crate) unsafe fn grayf16_to_rgba_u16_row<const BE: bool>(
y_plane: &[half::f16],
out: &mut [u16],
width: usize,
) {
use crate::row::scalar::grayf16 as scalar;
debug_assert!(y_plane.len() >= width);
debug_assert!(out.len() >= width * 4);
let mut x = 0usize;
while x + 4 <= width {
let mut buf = [0.0f32; 4];
unsafe {
widen_f16x4_sse_buf::<BE>(y_plane.as_ptr().add(x), buf.as_mut_ptr());
grayf32_to_rgba_u16_row::<GRAYF16_HOST_NATIVE_BE>(&buf, &mut out[x * 4..(x + 4) * 4], 4);
}
x += 4;
}
if x < width {
scalar::grayf16_to_rgba_u16_row::<BE>(
&y_plane[x..width],
&mut out[x * 4..width * 4],
width - x,
);
}
}
#[inline]
#[target_feature(enable = "sse4.1,f16c")]
pub(crate) unsafe fn grayf16_to_luma_row<const BE: bool>(
y_plane: &[half::f16],
out: &mut [u8],
width: usize,
) {
use crate::row::scalar::grayf16 as scalar;
debug_assert!(y_plane.len() >= width);
debug_assert!(out.len() >= width);
let mut x = 0usize;
while x + 4 <= width {
let mut buf = [0.0f32; 4];
unsafe {
widen_f16x4_sse_buf::<BE>(y_plane.as_ptr().add(x), buf.as_mut_ptr());
grayf32_to_luma_row::<GRAYF16_HOST_NATIVE_BE>(&buf, &mut out[x..x + 4], 4);
}
x += 4;
}
if x < width {
scalar::grayf16_to_luma_row::<BE>(&y_plane[x..width], &mut out[x..width], width - x);
}
}
#[inline]
#[target_feature(enable = "sse4.1,f16c")]
pub(crate) unsafe fn grayf16_to_luma_u16_row<const BE: bool>(
y_plane: &[half::f16],
out: &mut [u16],
width: usize,
) {
use crate::row::scalar::grayf16 as scalar;
debug_assert!(y_plane.len() >= width);
debug_assert!(out.len() >= width);
let mut x = 0usize;
while x + 4 <= width {
let mut buf = [0.0f32; 4];
unsafe {
widen_f16x4_sse_buf::<BE>(y_plane.as_ptr().add(x), buf.as_mut_ptr());
grayf32_to_luma_u16_row::<GRAYF16_HOST_NATIVE_BE>(&buf, &mut out[x..x + 4], 4);
}
x += 4;
}
if x < width {
scalar::grayf16_to_luma_u16_row::<BE>(&y_plane[x..width], &mut out[x..width], width - x);
}
}
#[inline]
#[target_feature(enable = "sse4.1,f16c")]
pub(crate) unsafe fn grayf16_to_hsv_row<const BE: bool>(
y_plane: &[half::f16],
h_out: &mut [u8],
s_out: &mut [u8],
v_out: &mut [u8],
width: usize,
) {
use crate::row::scalar::grayf16 as scalar;
debug_assert!(y_plane.len() >= width);
let mut x = 0usize;
while x + 4 <= width {
let mut buf = [0.0f32; 4];
unsafe {
widen_f16x4_sse_buf::<BE>(y_plane.as_ptr().add(x), buf.as_mut_ptr());
grayf32_to_hsv_row::<GRAYF16_HOST_NATIVE_BE>(
&buf,
&mut h_out[x..x + 4],
&mut s_out[x..x + 4],
&mut v_out[x..x + 4],
4,
);
}
x += 4;
}
if x < width {
scalar::grayf16_to_hsv_row::<BE>(
&y_plane[x..width],
&mut h_out[x..width],
&mut s_out[x..width],
&mut v_out[x..width],
width - x,
);
}
}
#[inline]
#[target_feature(enable = "sse4.1")]
pub(crate) unsafe fn yaf32_to_rgb_row<const BE: bool>(
packed: &[f32],
out: &mut [u8],
width: usize,
) {
use crate::row::scalar::yaf32 as scalar;
debug_assert!(packed.len() >= width * 2);
debug_assert!(out.len() >= width * 3);
if BE != HOST_NATIVE_BE {
return scalar::yaf32_to_rgb_row::<BE>(packed, out, width);
}
let mut x = 0usize;
while x + 4 <= width {
let mut ybuf = [0.0f32; 4];
unsafe {
let ya0 = _mm_loadu_ps(packed.as_ptr().add(x * 2));
let ya1 = _mm_loadu_ps(packed.as_ptr().add(x * 2 + 4));
_mm_storeu_ps(ybuf.as_mut_ptr(), _mm_shuffle_ps::<0x88>(ya0, ya1));
grayf32_to_rgb_row::<HOST_NATIVE_BE>(&ybuf, &mut out[x * 3..(x + 4) * 3], 4);
}
x += 4;
}
if x < width {
scalar::yaf32_to_rgb_row::<BE>(
&packed[x * 2..width * 2],
&mut out[x * 3..width * 3],
width - x,
);
}
}
#[inline]
#[target_feature(enable = "sse4.1")]
pub(crate) unsafe fn yaf32_to_rgba_row<const BE: bool>(
packed: &[f32],
out: &mut [u8],
width: usize,
) {
use crate::row::scalar::yaf32 as scalar;
debug_assert!(packed.len() >= width * 2);
debug_assert!(out.len() >= width * 4);
if BE != HOST_NATIVE_BE {
return scalar::yaf32_to_rgba_row::<BE>(packed, out, width);
}
let mut x = 0usize;
while x + 4 <= width {
let mut ybuf = [0.0f32; 4];
let mut abuf = [0.0f32; 4];
let mut a8 = [0u8; 4];
unsafe {
let ya0 = _mm_loadu_ps(packed.as_ptr().add(x * 2));
let ya1 = _mm_loadu_ps(packed.as_ptr().add(x * 2 + 4));
_mm_storeu_ps(ybuf.as_mut_ptr(), _mm_shuffle_ps::<0x88>(ya0, ya1));
_mm_storeu_ps(abuf.as_mut_ptr(), _mm_shuffle_ps::<0xDD>(ya0, ya1));
grayf32_to_rgba_row::<HOST_NATIVE_BE>(&ybuf, &mut out[x * 4..(x + 4) * 4], 4);
grayf32_to_luma_row::<HOST_NATIVE_BE>(&abuf, &mut a8, 4);
}
for i in 0..4 {
out[(x + i) * 4 + 3] = a8[i];
}
x += 4;
}
if x < width {
scalar::yaf32_to_rgba_row::<BE>(
&packed[x * 2..width * 2],
&mut out[x * 4..width * 4],
width - x,
);
}
}
#[inline]
#[target_feature(enable = "sse4.1")]
pub(crate) unsafe fn yaf32_to_rgb_u16_row<const BE: bool>(
packed: &[f32],
out: &mut [u16],
width: usize,
) {
use crate::row::scalar::yaf32 as scalar;
debug_assert!(packed.len() >= width * 2);
debug_assert!(out.len() >= width * 3);
if BE != HOST_NATIVE_BE {
return scalar::yaf32_to_rgb_u16_row::<BE>(packed, out, width);
}
let mut x = 0usize;
while x + 4 <= width {
let mut ybuf = [0.0f32; 4];
unsafe {
let ya0 = _mm_loadu_ps(packed.as_ptr().add(x * 2));
let ya1 = _mm_loadu_ps(packed.as_ptr().add(x * 2 + 4));
_mm_storeu_ps(ybuf.as_mut_ptr(), _mm_shuffle_ps::<0x88>(ya0, ya1));
grayf32_to_rgb_u16_row::<HOST_NATIVE_BE>(&ybuf, &mut out[x * 3..(x + 4) * 3], 4);
}
x += 4;
}
if x < width {
scalar::yaf32_to_rgb_u16_row::<BE>(
&packed[x * 2..width * 2],
&mut out[x * 3..width * 3],
width - x,
);
}
}
#[inline]
#[target_feature(enable = "sse4.1")]
pub(crate) unsafe fn yaf32_to_rgba_u16_row<const BE: bool>(
packed: &[f32],
out: &mut [u16],
width: usize,
) {
use crate::row::scalar::yaf32 as scalar;
debug_assert!(packed.len() >= width * 2);
debug_assert!(out.len() >= width * 4);
if BE != HOST_NATIVE_BE {
return scalar::yaf32_to_rgba_u16_row::<BE>(packed, out, width);
}
let mut x = 0usize;
while x + 4 <= width {
let mut ybuf = [0.0f32; 4];
let mut abuf = [0.0f32; 4];
let mut a16 = [0u16; 4];
unsafe {
let ya0 = _mm_loadu_ps(packed.as_ptr().add(x * 2));
let ya1 = _mm_loadu_ps(packed.as_ptr().add(x * 2 + 4));
_mm_storeu_ps(ybuf.as_mut_ptr(), _mm_shuffle_ps::<0x88>(ya0, ya1));
_mm_storeu_ps(abuf.as_mut_ptr(), _mm_shuffle_ps::<0xDD>(ya0, ya1));
grayf32_to_rgba_u16_row::<HOST_NATIVE_BE>(&ybuf, &mut out[x * 4..(x + 4) * 4], 4);
grayf32_to_luma_u16_row::<HOST_NATIVE_BE>(&abuf, &mut a16, 4);
}
for i in 0..4 {
out[(x + i) * 4 + 3] = a16[i];
}
x += 4;
}
if x < width {
scalar::yaf32_to_rgba_u16_row::<BE>(
&packed[x * 2..width * 2],
&mut out[x * 4..width * 4],
width - x,
);
}
}
#[inline]
#[target_feature(enable = "sse4.1")]
pub(crate) unsafe fn yaf32_to_luma_row<const BE: bool>(
packed: &[f32],
out: &mut [u8],
width: usize,
) {
use crate::row::scalar::yaf32 as scalar;
debug_assert!(packed.len() >= width * 2);
debug_assert!(out.len() >= width);
if BE != HOST_NATIVE_BE {
return scalar::yaf32_to_luma_row::<BE>(packed, out, width);
}
let mut x = 0usize;
while x + 4 <= width {
let mut ybuf = [0.0f32; 4];
unsafe {
let ya0 = _mm_loadu_ps(packed.as_ptr().add(x * 2));
let ya1 = _mm_loadu_ps(packed.as_ptr().add(x * 2 + 4));
_mm_storeu_ps(ybuf.as_mut_ptr(), _mm_shuffle_ps::<0x88>(ya0, ya1));
grayf32_to_luma_row::<HOST_NATIVE_BE>(&ybuf, &mut out[x..x + 4], 4);
}
x += 4;
}
if x < width {
scalar::yaf32_to_luma_row::<BE>(&packed[x * 2..width * 2], &mut out[x..width], width - x);
}
}
#[inline]
#[target_feature(enable = "sse4.1")]
pub(crate) unsafe fn yaf32_to_luma_u16_row<const BE: bool>(
packed: &[f32],
out: &mut [u16],
width: usize,
) {
use crate::row::scalar::yaf32 as scalar;
debug_assert!(packed.len() >= width * 2);
debug_assert!(out.len() >= width);
if BE != HOST_NATIVE_BE {
return scalar::yaf32_to_luma_u16_row::<BE>(packed, out, width);
}
let mut x = 0usize;
while x + 4 <= width {
let mut ybuf = [0.0f32; 4];
unsafe {
let ya0 = _mm_loadu_ps(packed.as_ptr().add(x * 2));
let ya1 = _mm_loadu_ps(packed.as_ptr().add(x * 2 + 4));
_mm_storeu_ps(ybuf.as_mut_ptr(), _mm_shuffle_ps::<0x88>(ya0, ya1));
grayf32_to_luma_u16_row::<HOST_NATIVE_BE>(&ybuf, &mut out[x..x + 4], 4);
}
x += 4;
}
if x < width {
scalar::yaf32_to_luma_u16_row::<BE>(&packed[x * 2..width * 2], &mut out[x..width], width - x);
}
}
#[inline]
#[target_feature(enable = "sse4.1")]
pub(crate) unsafe fn yaf32_to_hsv_row<const BE: bool>(
packed: &[f32],
h_out: &mut [u8],
s_out: &mut [u8],
v_out: &mut [u8],
width: usize,
) {
use crate::row::scalar::yaf32 as scalar;
debug_assert!(packed.len() >= width * 2);
if BE != HOST_NATIVE_BE {
return scalar::yaf32_to_hsv_row::<BE>(packed, h_out, s_out, v_out, width);
}
let mut x = 0usize;
while x + 4 <= width {
let mut ybuf = [0.0f32; 4];
unsafe {
let ya0 = _mm_loadu_ps(packed.as_ptr().add(x * 2));
let ya1 = _mm_loadu_ps(packed.as_ptr().add(x * 2 + 4));
_mm_storeu_ps(ybuf.as_mut_ptr(), _mm_shuffle_ps::<0x88>(ya0, ya1));
grayf32_to_hsv_row::<HOST_NATIVE_BE>(
&ybuf,
&mut h_out[x..x + 4],
&mut s_out[x..x + 4],
&mut v_out[x..x + 4],
4,
);
}
x += 4;
}
if x < width {
scalar::yaf32_to_hsv_row::<BE>(
&packed[x * 2..width * 2],
&mut h_out[x..width],
&mut s_out[x..width],
&mut v_out[x..width],
width - x,
);
}
}
#[inline]
#[target_feature(enable = "sse4.1,f16c")]
pub(crate) unsafe fn yaf16_to_rgb_row<const BE: bool>(
packed: &[half::f16],
out: &mut [u8],
width: usize,
) {
use crate::row::scalar::yaf16 as scalar;
debug_assert!(packed.len() >= width * 2);
debug_assert!(out.len() >= width * 3);
let mut x = 0usize;
while x + 4 <= width {
let mut buf = [0.0f32; 8];
unsafe {
widen_f16x4_sse_buf::<BE>(packed.as_ptr().add(x * 2), buf.as_mut_ptr());
widen_f16x4_sse_buf::<BE>(packed.as_ptr().add(x * 2 + 4), buf.as_mut_ptr().add(4));
yaf32_to_rgb_row::<HOST_NATIVE_BE>(&buf, &mut out[x * 3..(x + 4) * 3], 4);
}
x += 4;
}
if x < width {
scalar::yaf16_to_rgb_row::<BE>(
&packed[x * 2..width * 2],
&mut out[x * 3..width * 3],
width - x,
);
}
}
#[inline]
#[target_feature(enable = "sse4.1,f16c")]
pub(crate) unsafe fn yaf16_to_rgba_row<const BE: bool>(
packed: &[half::f16],
out: &mut [u8],
width: usize,
) {
use crate::row::scalar::yaf16 as scalar;
debug_assert!(packed.len() >= width * 2);
debug_assert!(out.len() >= width * 4);
let mut x = 0usize;
while x + 4 <= width {
let mut buf = [0.0f32; 8];
unsafe {
widen_f16x4_sse_buf::<BE>(packed.as_ptr().add(x * 2), buf.as_mut_ptr());
widen_f16x4_sse_buf::<BE>(packed.as_ptr().add(x * 2 + 4), buf.as_mut_ptr().add(4));
yaf32_to_rgba_row::<HOST_NATIVE_BE>(&buf, &mut out[x * 4..(x + 4) * 4], 4);
}
x += 4;
}
if x < width {
scalar::yaf16_to_rgba_row::<BE>(
&packed[x * 2..width * 2],
&mut out[x * 4..width * 4],
width - x,
);
}
}
#[inline]
#[target_feature(enable = "sse4.1,f16c")]
pub(crate) unsafe fn yaf16_to_rgb_u16_row<const BE: bool>(
packed: &[half::f16],
out: &mut [u16],
width: usize,
) {
use crate::row::scalar::yaf16 as scalar;
debug_assert!(packed.len() >= width * 2);
debug_assert!(out.len() >= width * 3);
let mut x = 0usize;
while x + 4 <= width {
let mut buf = [0.0f32; 8];
unsafe {
widen_f16x4_sse_buf::<BE>(packed.as_ptr().add(x * 2), buf.as_mut_ptr());
widen_f16x4_sse_buf::<BE>(packed.as_ptr().add(x * 2 + 4), buf.as_mut_ptr().add(4));
yaf32_to_rgb_u16_row::<HOST_NATIVE_BE>(&buf, &mut out[x * 3..(x + 4) * 3], 4);
}
x += 4;
}
if x < width {
scalar::yaf16_to_rgb_u16_row::<BE>(
&packed[x * 2..width * 2],
&mut out[x * 3..width * 3],
width - x,
);
}
}
#[inline]
#[target_feature(enable = "sse4.1,f16c")]
pub(crate) unsafe fn yaf16_to_rgba_u16_row<const BE: bool>(
packed: &[half::f16],
out: &mut [u16],
width: usize,
) {
use crate::row::scalar::yaf16 as scalar;
debug_assert!(packed.len() >= width * 2);
debug_assert!(out.len() >= width * 4);
let mut x = 0usize;
while x + 4 <= width {
let mut buf = [0.0f32; 8];
unsafe {
widen_f16x4_sse_buf::<BE>(packed.as_ptr().add(x * 2), buf.as_mut_ptr());
widen_f16x4_sse_buf::<BE>(packed.as_ptr().add(x * 2 + 4), buf.as_mut_ptr().add(4));
yaf32_to_rgba_u16_row::<HOST_NATIVE_BE>(&buf, &mut out[x * 4..(x + 4) * 4], 4);
}
x += 4;
}
if x < width {
scalar::yaf16_to_rgba_u16_row::<BE>(
&packed[x * 2..width * 2],
&mut out[x * 4..width * 4],
width - x,
);
}
}
#[inline]
#[target_feature(enable = "sse4.1,f16c")]
pub(crate) unsafe fn yaf16_to_luma_row<const BE: bool>(
packed: &[half::f16],
out: &mut [u8],
width: usize,
) {
use crate::row::scalar::yaf16 as scalar;
debug_assert!(packed.len() >= width * 2);
debug_assert!(out.len() >= width);
let mut x = 0usize;
while x + 4 <= width {
let mut buf = [0.0f32; 8];
unsafe {
widen_f16x4_sse_buf::<BE>(packed.as_ptr().add(x * 2), buf.as_mut_ptr());
widen_f16x4_sse_buf::<BE>(packed.as_ptr().add(x * 2 + 4), buf.as_mut_ptr().add(4));
yaf32_to_luma_row::<HOST_NATIVE_BE>(&buf, &mut out[x..x + 4], 4);
}
x += 4;
}
if x < width {
scalar::yaf16_to_luma_row::<BE>(&packed[x * 2..width * 2], &mut out[x..width], width - x);
}
}
#[inline]
#[target_feature(enable = "sse4.1,f16c")]
pub(crate) unsafe fn yaf16_to_luma_u16_row<const BE: bool>(
packed: &[half::f16],
out: &mut [u16],
width: usize,
) {
use crate::row::scalar::yaf16 as scalar;
debug_assert!(packed.len() >= width * 2);
debug_assert!(out.len() >= width);
let mut x = 0usize;
while x + 4 <= width {
let mut buf = [0.0f32; 8];
unsafe {
widen_f16x4_sse_buf::<BE>(packed.as_ptr().add(x * 2), buf.as_mut_ptr());
widen_f16x4_sse_buf::<BE>(packed.as_ptr().add(x * 2 + 4), buf.as_mut_ptr().add(4));
yaf32_to_luma_u16_row::<HOST_NATIVE_BE>(&buf, &mut out[x..x + 4], 4);
}
x += 4;
}
if x < width {
scalar::yaf16_to_luma_u16_row::<BE>(&packed[x * 2..width * 2], &mut out[x..width], width - x);
}
}
#[inline]
#[target_feature(enable = "sse4.1,f16c")]
pub(crate) unsafe fn yaf16_to_hsv_row<const BE: bool>(
packed: &[half::f16],
h_out: &mut [u8],
s_out: &mut [u8],
v_out: &mut [u8],
width: usize,
) {
use crate::row::scalar::yaf16 as scalar;
debug_assert!(packed.len() >= width * 2);
let mut x = 0usize;
while x + 4 <= width {
let mut buf = [0.0f32; 8];
unsafe {
widen_f16x4_sse_buf::<BE>(packed.as_ptr().add(x * 2), buf.as_mut_ptr());
widen_f16x4_sse_buf::<BE>(packed.as_ptr().add(x * 2 + 4), buf.as_mut_ptr().add(4));
yaf32_to_hsv_row::<HOST_NATIVE_BE>(
&buf,
&mut h_out[x..x + 4],
&mut s_out[x..x + 4],
&mut v_out[x..x + 4],
4,
);
}
x += 4;
}
if x < width {
scalar::yaf16_to_hsv_row::<BE>(
&packed[x * 2..width * 2],
&mut h_out[x..width],
&mut s_out[x..width],
&mut v_out[x..width],
width - x,
);
}
}