#![cfg(feature = "avx_luts")]
use crate::conversions::LutBarycentricReduction;
use crate::conversions::avx::interpolator_q0_15::*;
use crate::conversions::interpolator::BarycentricWeight;
use crate::transform::PointeeSizeExpressible;
use crate::{CmsError, DataColorSpace, InterpolationMethod, Layout, TransformExecutor};
use num_traits::AsPrimitive;
use std::arch::x86_64::*;
use std::marker::PhantomData;
pub(crate) struct TransformLut4To3AvxQ0_15<
T,
U,
const LAYOUT: u8,
const GRID_SIZE: usize,
const BIT_DEPTH: usize,
const BINS: usize,
const BARYCENTRIC_BINS: usize,
> {
pub(crate) lut: Vec<AvxAlignedI16>,
pub(crate) _phantom: PhantomData<T>,
pub(crate) _phantom1: PhantomData<U>,
pub(crate) interpolation_method: InterpolationMethod,
pub(crate) weights: Box<[BarycentricWeight<i16>; BINS]>,
pub(crate) color_space: DataColorSpace,
pub(crate) is_linear: bool,
}
impl<
T: Copy + AsPrimitive<f32> + Default + PointeeSizeExpressible,
U: AsPrimitive<usize>,
const LAYOUT: u8,
const GRID_SIZE: usize,
const BIT_DEPTH: usize,
const BINS: usize,
const BARYCENTRIC_BINS: usize,
> TransformLut4To3AvxQ0_15<T, U, LAYOUT, GRID_SIZE, BIT_DEPTH, BINS, BARYCENTRIC_BINS>
where
f32: AsPrimitive<T>,
u32: AsPrimitive<T>,
(): LutBarycentricReduction<T, U>,
{
#[allow(unused_unsafe)]
#[target_feature(enable = "avx2")]
unsafe fn transform_chunk(
&self,
src: &[T],
dst: &mut [T],
interpolator: Box<dyn AvxMdInterpolationQ0_15Double + Send + Sync>,
) {
unsafe {
let cn = Layout::from(LAYOUT);
let channels = cn.channels();
let grid_size = GRID_SIZE as i32;
let grid_size3 = grid_size * grid_size * grid_size;
let f_value_scale = _mm_set1_ps(1. / ((1 << 14i32) - 1) as f32);
let max_value = ((1u32 << BIT_DEPTH) - 1).as_();
let v_max_scale = if T::FINITE {
_mm_set1_epi16(((1i32 << BIT_DEPTH) - 1) as i16)
} else {
_mm_set1_epi16(((1i32 << 14i32) - 1) as i16)
};
for (src, dst) in src.chunks_exact(4).zip(dst.chunks_exact_mut(channels)) {
let c = <() as LutBarycentricReduction<T, U>>::reduce::<BIT_DEPTH, BARYCENTRIC_BINS>(
src[0],
);
let m = <() as LutBarycentricReduction<T, U>>::reduce::<BIT_DEPTH, BARYCENTRIC_BINS>(
src[1],
);
let y = <() as LutBarycentricReduction<T, U>>::reduce::<BIT_DEPTH, BARYCENTRIC_BINS>(
src[2],
);
let k = <() as LutBarycentricReduction<T, U>>::reduce::<BIT_DEPTH, BARYCENTRIC_BINS>(
src[3],
);
let k_weights = self.weights[k.as_()];
let w: i32 = k_weights.x;
let w_n: i32 = k_weights.x_n;
const Q: i16 = ((1i32 << 15) - 1) as i16;
let t: i16 = k_weights.w;
let t_n: i16 = Q - t;
let table1 = &self.lut[(w * grid_size3) as usize..];
let table2 = &self.lut[(w_n * grid_size3) as usize..];
let v = interpolator.inter3_sse(
table1,
table2,
c.as_(),
m.as_(),
y.as_(),
self.weights.as_slice(),
);
let (a0, b0) = (v.0.v, v.1.v);
let hp = _mm_mulhrs_epi16(_mm_set1_epi16(t_n), a0);
let v = _mm_add_epi16(hp, _mm_mulhrs_epi16(b0, _mm_set1_epi16(t)));
if T::FINITE {
let mut o = _mm_max_epi16(v, _mm_setzero_si128());
o = _mm_min_epi16(o, v_max_scale);
let x = _mm_extract_epi16::<0>(o);
let y = _mm_extract_epi16::<1>(o);
let z = _mm_extract_epi16::<2>(o);
dst[cn.r_i()] = (x as u32).as_();
dst[cn.g_i()] = (y as u32).as_();
dst[cn.b_i()] = (z as u32).as_();
} else {
let mut r = _mm_cvtepi32_ps(_mm_cvtepi16_epi32(v));
r = _mm_mul_ps(r, f_value_scale);
dst[cn.r_i()] = f32::from_bits(_mm_extract_ps::<0>(r) as u32).as_();
dst[cn.g_i()] = f32::from_bits(_mm_extract_ps::<1>(r) as u32).as_();
dst[cn.b_i()] = f32::from_bits(_mm_extract_ps::<2>(r) as u32).as_();
}
if channels == 4 {
dst[cn.a_i()] = max_value;
}
}
}
}
}
impl<
T: Copy + AsPrimitive<f32> + Default + PointeeSizeExpressible,
U: AsPrimitive<usize>,
const LAYOUT: u8,
const GRID_SIZE: usize,
const BIT_DEPTH: usize,
const BINS: usize,
const BARYCENTRIC_BINS: usize,
> TransformExecutor<T>
for TransformLut4To3AvxQ0_15<T, U, LAYOUT, GRID_SIZE, BIT_DEPTH, BINS, BARYCENTRIC_BINS>
where
f32: AsPrimitive<T>,
u32: AsPrimitive<T>,
(): LutBarycentricReduction<T, U>,
{
fn transform(&self, src: &[T], dst: &mut [T]) -> Result<(), CmsError> {
let cn = Layout::from(LAYOUT);
let channels = cn.channels();
if src.len() % 4 != 0 {
return Err(CmsError::LaneMultipleOfChannels);
}
if dst.len() % channels != 0 {
return Err(CmsError::LaneMultipleOfChannels);
}
let src_chunks = src.len() / 4;
let dst_chunks = dst.len() / channels;
if src_chunks != dst_chunks {
return Err(CmsError::LaneSizeMismatch);
}
unsafe {
if self.color_space == DataColorSpace::Lab
|| (self.is_linear && self.color_space == DataColorSpace::Rgb)
|| self.color_space == DataColorSpace::Xyz
{
self.transform_chunk(src, dst, Box::new(TrilinearAvxQ0_15Double::<GRID_SIZE> {}));
} else {
match self.interpolation_method {
#[cfg(feature = "options")]
InterpolationMethod::Tetrahedral => {
self.transform_chunk(
src,
dst,
Box::new(TetrahedralAvxQ0_15Double::<GRID_SIZE> {}),
);
}
#[cfg(feature = "options")]
InterpolationMethod::Pyramid => {
self.transform_chunk(
src,
dst,
Box::new(PyramidAvxFmaQ0_15Double::<GRID_SIZE> {}),
);
}
#[cfg(feature = "options")]
InterpolationMethod::Prism => {
self.transform_chunk(
src,
dst,
Box::new(PrismaticAvxQ0_15Double::<GRID_SIZE> {}),
);
}
InterpolationMethod::Linear => {
self.transform_chunk(
src,
dst,
Box::new(TrilinearAvxQ0_15Double::<GRID_SIZE> {}),
);
}
}
}
}
Ok(())
}
}