#![cfg(feature = "sse_luts")]
use crate::conversions::LutBarycentricReduction;
use crate::conversions::interpolator::BarycentricWeight;
use crate::conversions::lut_transforms::Lut3x3Factory;
use crate::conversions::sse::assert_barycentric_lut_size_precondition;
use crate::conversions::sse::interpolator::*;
use crate::conversions::sse::interpolator_q0_15::SseAlignedI16x4;
use crate::conversions::sse::t_lut3_to_3_q0_15::TransformLut3x3SseQ0_15;
use crate::transform::PointeeSizeExpressible;
use crate::{
BarycentricWeightScale, CmsError, DataColorSpace, InterpolationMethod, Layout,
TransformExecutor, TransformOptions,
};
use num_traits::AsPrimitive;
#[cfg(target_arch = "x86")]
use std::arch::x86::*;
#[cfg(target_arch = "x86_64")]
use std::arch::x86_64::*;
use std::marker::PhantomData;
use std::sync::Arc;
struct TransformLut3x3Sse<
T,
U,
const SRC_LAYOUT: u8,
const DST_LAYOUT: u8,
const GRID_SIZE: usize,
const BIT_DEPTH: usize,
const BINS: usize,
const BARYCENTRIC_BINS: usize,
> {
lut: Vec<SseAlignedF32>,
_phantom: PhantomData<T>,
_phantom2: PhantomData<U>,
interpolation_method: InterpolationMethod,
weights: Box<[BarycentricWeight<f32>; BINS]>,
color_space: DataColorSpace,
is_linear: bool,
}
impl<
T: Copy + AsPrimitive<f32> + Default + PointeeSizeExpressible,
U: AsPrimitive<usize>,
const SRC_LAYOUT: u8,
const DST_LAYOUT: u8,
const GRID_SIZE: usize,
const BIT_DEPTH: usize,
const BINS: usize,
const BARYCENTRIC_BINS: usize,
> TransformLut3x3Sse<T, U, SRC_LAYOUT, DST_LAYOUT, GRID_SIZE, BIT_DEPTH, BINS, BARYCENTRIC_BINS>
where
f32: AsPrimitive<T>,
u32: AsPrimitive<T>,
(): LutBarycentricReduction<T, U>,
{
#[allow(unused_unsafe)]
#[target_feature(enable = "sse4.1")]
unsafe fn transform_chunk(
&self,
src: &[T],
dst: &mut [T],
interpolator: Box<dyn SseMdInterpolation + Send + Sync>,
) {
let src_cn = Layout::from(SRC_LAYOUT);
let src_channels = src_cn.channels();
let dst_cn = Layout::from(DST_LAYOUT);
let dst_channels = dst_cn.channels();
let value_scale = unsafe { _mm_set1_ps(((1 << BIT_DEPTH) - 1) as f32) };
let max_value = ((1u32 << BIT_DEPTH) - 1).as_();
for (src, dst) in src
.chunks_exact(src_channels)
.zip(dst.chunks_exact_mut(dst_channels))
{
let x = <() as LutBarycentricReduction<T, U>>::reduce::<BIT_DEPTH, BARYCENTRIC_BINS>(
src[src_cn.r_i()],
);
let y = <() as LutBarycentricReduction<T, U>>::reduce::<BIT_DEPTH, BARYCENTRIC_BINS>(
src[src_cn.g_i()],
);
let z = <() as LutBarycentricReduction<T, U>>::reduce::<BIT_DEPTH, BARYCENTRIC_BINS>(
src[src_cn.b_i()],
);
let a = if src_channels == 4 {
src[src_cn.a_i()]
} else {
max_value
};
let v = interpolator.inter3_sse(
&self.lut,
x.as_(),
y.as_(),
z.as_(),
self.weights.as_slice(),
);
if T::FINITE {
unsafe {
let mut r = _mm_mul_ps(v.v, value_scale);
r = _mm_max_ps(r, _mm_setzero_ps());
r = _mm_min_ps(r, value_scale);
let jvz = _mm_cvtps_epi32(r);
let x = _mm_extract_epi32::<0>(jvz);
let y = _mm_extract_epi32::<1>(jvz);
let z = _mm_extract_epi32::<2>(jvz);
dst[dst_cn.r_i()] = (x as u32).as_();
dst[dst_cn.g_i()] = (y as u32).as_();
dst[dst_cn.b_i()] = (z as u32).as_();
}
} else {
unsafe {
dst[dst_cn.r_i()] = f32::from_bits(_mm_extract_ps::<0>(v.v) as u32).as_();
dst[dst_cn.g_i()] = f32::from_bits(_mm_extract_ps::<1>(v.v) as u32).as_();
dst[dst_cn.b_i()] = f32::from_bits(_mm_extract_ps::<2>(v.v) as u32).as_();
}
}
if dst_channels == 4 {
dst[dst_cn.a_i()] = a;
}
}
}
}
impl<
T: Copy + AsPrimitive<f32> + Default + PointeeSizeExpressible,
U: AsPrimitive<usize>,
const SRC_LAYOUT: u8,
const DST_LAYOUT: u8,
const GRID_SIZE: usize,
const BIT_DEPTH: usize,
const BINS: usize,
const BARYCENTRIC_BINS: usize,
> TransformExecutor<T>
for TransformLut3x3Sse<
T,
U,
SRC_LAYOUT,
DST_LAYOUT,
GRID_SIZE,
BIT_DEPTH,
BINS,
BARYCENTRIC_BINS,
>
where
f32: AsPrimitive<T>,
u32: AsPrimitive<T>,
(): LutBarycentricReduction<T, U>,
{
fn transform(&self, src: &[T], dst: &mut [T]) -> Result<(), CmsError> {
let src_cn = Layout::from(SRC_LAYOUT);
let src_channels = src_cn.channels();
let dst_cn = Layout::from(DST_LAYOUT);
let dst_channels = dst_cn.channels();
if src.len() % src_channels != 0 {
return Err(CmsError::LaneMultipleOfChannels);
}
if dst.len() % dst_channels != 0 {
return Err(CmsError::LaneMultipleOfChannels);
}
let src_chunks = src.len() / src_channels;
let dst_chunks = dst.len() / dst_channels;
if src_chunks != dst_chunks {
return Err(CmsError::LaneSizeMismatch);
}
unsafe {
if self.color_space == DataColorSpace::Lab
|| (self.is_linear && self.color_space == DataColorSpace::Rgb)
|| self.color_space == DataColorSpace::Xyz
{
self.transform_chunk(src, dst, Box::new(TrilinearSse::<GRID_SIZE> {}));
} else {
match self.interpolation_method {
#[cfg(feature = "options")]
InterpolationMethod::Tetrahedral => {
self.transform_chunk(src, dst, Box::new(TetrahedralSse::<GRID_SIZE> {}));
}
#[cfg(feature = "options")]
InterpolationMethod::Pyramid => {
self.transform_chunk(src, dst, Box::new(PyramidalSse::<GRID_SIZE> {}));
}
#[cfg(feature = "options")]
InterpolationMethod::Prism => {
self.transform_chunk(src, dst, Box::new(PrismaticSse::<GRID_SIZE> {}));
}
InterpolationMethod::Linear => {
self.transform_chunk(src, dst, Box::new(TrilinearSse::<GRID_SIZE> {}));
}
}
}
}
Ok(())
}
}
pub(crate) struct SseLut3x3Factory {}
impl Lut3x3Factory for SseLut3x3Factory {
fn make_transform_3x3<
T: Copy + AsPrimitive<f32> + Default + PointeeSizeExpressible + 'static + Send + Sync,
const SRC_LAYOUT: u8,
const DST_LAYOUT: u8,
const GRID_SIZE: usize,
const BIT_DEPTH: usize,
>(
lut: Vec<f32>,
options: TransformOptions,
color_space: DataColorSpace,
is_linear: bool,
) -> Arc<dyn TransformExecutor<T> + Sync + Send>
where
f32: AsPrimitive<T>,
u32: AsPrimitive<T>,
(): LutBarycentricReduction<T, u8>,
(): LutBarycentricReduction<T, u16>,
{
if options.prefer_fixed_point && BIT_DEPTH < 16 {
let q: f32 = if T::FINITE {
((1i32 << BIT_DEPTH as i32) - 1) as f32
} else {
((1i32 << 14i32) - 1) as f32
};
let lut = lut
.chunks_exact(3)
.map(|x| {
SseAlignedI16x4([
(x[0] * q).round() as i16,
(x[1] * q).round() as i16,
(x[2] * q).round() as i16,
0,
])
})
.collect::<Vec<_>>();
return match options.barycentric_weight_scale {
BarycentricWeightScale::Low => {
let bins = BarycentricWeight::<i16>::create_ranged_256::<GRID_SIZE>();
assert_barycentric_lut_size_precondition::<i16, GRID_SIZE>(bins.as_slice());
Arc::new(TransformLut3x3SseQ0_15::<
T,
u8,
SRC_LAYOUT,
DST_LAYOUT,
GRID_SIZE,
BIT_DEPTH,
256,
256,
> {
lut,
_phantom: PhantomData,
_phantom2: PhantomData,
interpolation_method: options.interpolation_method,
weights: bins,
color_space,
is_linear,
})
}
#[cfg(feature = "options")]
BarycentricWeightScale::High => {
let bins = BarycentricWeight::<i16>::create_binned::<GRID_SIZE, 65536>();
assert_barycentric_lut_size_precondition::<i16, GRID_SIZE>(bins.as_slice());
Arc::new(TransformLut3x3SseQ0_15::<
T,
u16,
SRC_LAYOUT,
DST_LAYOUT,
GRID_SIZE,
BIT_DEPTH,
65536,
65536,
> {
lut,
_phantom: PhantomData,
_phantom2: PhantomData,
interpolation_method: options.interpolation_method,
weights: bins,
color_space,
is_linear,
})
}
};
}
let lut = lut
.chunks_exact(3)
.map(|x| SseAlignedF32([x[0], x[1], x[2], 0f32]))
.collect::<Vec<_>>();
match options.barycentric_weight_scale {
BarycentricWeightScale::Low => {
let bins = BarycentricWeight::<f32>::create_ranged_256::<GRID_SIZE>();
assert_barycentric_lut_size_precondition::<f32, GRID_SIZE>(bins.as_slice());
Arc::new(TransformLut3x3Sse::<
T,
u8,
SRC_LAYOUT,
DST_LAYOUT,
GRID_SIZE,
BIT_DEPTH,
256,
256,
> {
lut,
_phantom: PhantomData,
_phantom2: PhantomData,
interpolation_method: options.interpolation_method,
weights: bins,
color_space,
is_linear,
})
}
#[cfg(feature = "options")]
BarycentricWeightScale::High => {
let bins = BarycentricWeight::<f32>::create_binned::<GRID_SIZE, 65536>();
assert_barycentric_lut_size_precondition::<f32, GRID_SIZE>(bins.as_slice());
Arc::new(TransformLut3x3Sse::<
T,
u16,
SRC_LAYOUT,
DST_LAYOUT,
GRID_SIZE,
BIT_DEPTH,
65536,
65536,
> {
lut,
_phantom: PhantomData,
_phantom2: PhantomData,
interpolation_method: options.interpolation_method,
weights: bins,
color_space,
is_linear,
})
}
}
}
}