use crate::xyb::*;
use std::arch::x86_64::*;
#[inline]
#[target_feature(enable = "avx2")]
#[cfg_attr(target_feature = "fma", target_feature(enable = "fma"))]
fn xyb_to_rgb_avx2(m: &XybMatrix, x: __m256, y: __m256, b: __m256) -> (__m256, __m256, __m256) {
let cube = |value: __m256| {
let value = _mm256_sub_ps(value, _mm256_set1_ps(NEG_BIAS_CBRT));
_mm256_sub_ps(
_mm256_mul_ps(_mm256_mul_ps(value, value), value),
_mm256_set1_ps(OPSIN_BIAS),
)
};
let m0 = cube(_mm256_add_ps(y, x));
let m1 = cube(_mm256_sub_ps(y, x));
let m2 = cube(b);
let row = |offset: usize| {
let value = _mm256_mul_ps(m2, _mm256_set1_ps(m.inv[offset + 2]));
#[cfg(target_feature = "fma")]
{
let value = _mm256_fmadd_ps(m1, _mm256_set1_ps(m.inv[offset + 1]), value);
_mm256_fmadd_ps(m0, _mm256_set1_ps(m.inv[offset]), value)
}
#[cfg(not(target_feature = "fma"))]
{
_mm256_add_ps(
_mm256_mul_ps(m0, _mm256_set1_ps(m.inv[offset])),
_mm256_add_ps(_mm256_mul_ps(m1, _mm256_set1_ps(m.inv[offset + 1])), value),
)
}
};
(row(0), row(3), row(6))
}
#[target_feature(enable = "avx2")]
#[cfg_attr(target_feature = "fma", target_feature(enable = "fma"))]
pub(crate) unsafe fn to_linear_rows_avx2(
m: &XybMatrix,
input: [&[f32]; 3],
output: [&mut [f32]; 3],
) -> usize {
let [r, g, b] = output;
let done = unsafe { to_linear_rows::<4>(m, input, [r, g, b]) };
done + unsafe {
to_linear_rows::<1>(
m,
input.map(|row| &row[done..]),
[&mut r[done..], &mut g[done..], &mut b[done..]],
)
}
}
#[inline]
#[target_feature(enable = "avx2")]
#[cfg_attr(target_feature = "fma", target_feature(enable = "fma"))]
unsafe fn to_linear_rows<const N: usize>(
m: &XybMatrix,
[x, y, b]: [&[f32]; 3],
[r, g, out_b]: [&mut [f32]; 3],
) -> usize {
const LANES: usize = 8;
let done = r.len() / (N * LANES) * (N * LANES);
for start in (0..done).step_by(N * LANES) {
for lane in 0..N {
let i = start + lane * LANES;
unsafe {
let xv = _mm256_loadu_ps(x.as_ptr().add(i));
let yv = _mm256_loadu_ps(y.as_ptr().add(i));
let bv = _mm256_loadu_ps(b.as_ptr().add(i));
let (rv, gv, bv) = xyb_to_rgb_avx2(m, xv, yv, bv);
_mm256_storeu_ps(r.as_mut_ptr().add(i), rv);
_mm256_storeu_ps(g.as_mut_ptr().add(i), gv);
_mm256_storeu_ps(out_b.as_mut_ptr().add(i), bv);
}
}
}
done
}
#[inline]
#[target_feature(enable = "avx2,fma")]
fn mul_add_avx2<const FUSED: bool>(a: __m256, b: __m256, c: __m256) -> __m256 {
if FUSED {
_mm256_fmadd_ps(a, b, c)
} else {
_mm256_add_ps(_mm256_mul_ps(a, b), c)
}
}
#[inline]
#[target_feature(enable = "avx2,fma")]
fn halley_cbrt_avx2<const SCALAR_ORDER: bool, const FUSED: bool>(x: __m256, a: __m256) -> __m256 {
let tx = _mm256_mul_ps(_mm256_mul_ps(x, x), x);
let two = _mm256_set1_ps(2.0);
let num = mul_add_avx2::<FUSED>(a, two, tx);
let den = mul_add_avx2::<FUSED>(tx, two, a);
if SCALAR_ORDER {
_mm256_div_ps(_mm256_mul_ps(x, num), den)
} else {
_mm256_mul_ps(x, _mm256_div_ps(num, den))
}
}
#[inline]
#[target_feature(enable = "avx2")]
fn integer_pow_1_3_avx2<const SCALAR_ORDER: bool>(hx: __m256i) -> __m256i {
let scale = _mm256_set1_epi32(if SCALAR_ORDER {
0xaaaa_aaabu32 as i32
} else {
341
});
let even64 = _mm256_mul_epu32(hx, scale);
let even = if SCALAR_ORDER {
_mm256_srli_epi64::<33>(even64)
} else {
_mm256_srli_epi64::<10>(even64)
};
let odd_src = _mm256_srli_epi64::<32>(hx);
let odd64 = _mm256_mul_epu32(odd_src, scale);
let odd = if SCALAR_ORDER {
_mm256_srli_epi64::<33>(odd64)
} else {
_mm256_srli_epi64::<10>(odd64)
};
let odd = _mm256_slli_epi64::<32>(odd);
_mm256_or_si256(even, odd)
}
#[inline]
#[target_feature(enable = "avx2")]
fn cbrt_seed_positive_avx2<const SCALAR_ORDER: bool>(x: __m256) -> __m256 {
let ui = _mm256_castps_si256(x);
let hx = _mm256_and_si256(ui, _mm256_set1_epi32(0x7fff_ffff));
let hx = _mm256_add_epi32(
integer_pow_1_3_avx2::<SCALAR_ORDER>(hx),
_mm256_set1_epi32(709958130),
);
_mm256_castsi256_ps(hx)
}
#[inline]
#[target_feature(enable = "avx2,fma")]
pub(crate) fn vcbrt_fast3_positive_avx2(
a0: __m256,
a1: __m256,
a2: __m256,
) -> (__m256, __m256, __m256) {
cbrt3_positive_avx2::<false, true>(a0, a1, a2)
}
#[inline]
#[target_feature(enable = "avx2,fma")]
fn cbrt3_positive_avx2<const SCALAR_ORDER: bool, const FUSED: bool>(
a0: __m256,
a1: __m256,
a2: __m256,
) -> (__m256, __m256, __m256) {
let mut x0 = cbrt_seed_positive_avx2::<SCALAR_ORDER>(a0);
let mut x1 = cbrt_seed_positive_avx2::<SCALAR_ORDER>(a1);
let mut x2 = cbrt_seed_positive_avx2::<SCALAR_ORDER>(a2);
x0 = halley_cbrt_avx2::<SCALAR_ORDER, FUSED>(x0, a0);
x1 = halley_cbrt_avx2::<SCALAR_ORDER, FUSED>(x1, a1);
x2 = halley_cbrt_avx2::<SCALAR_ORDER, FUSED>(x2, a2);
x0 = halley_cbrt_avx2::<SCALAR_ORDER, FUSED>(x0, a0);
x1 = halley_cbrt_avx2::<SCALAR_ORDER, FUSED>(x1, a1);
x2 = halley_cbrt_avx2::<SCALAR_ORDER, FUSED>(x2, a2);
let zero = _mm256_setzero_ps();
x0 = _mm256_blendv_ps(x0, zero, _mm256_cmp_ps::<_CMP_EQ_OQ>(a0, zero));
x1 = _mm256_blendv_ps(x1, zero, _mm256_cmp_ps::<_CMP_EQ_OQ>(a1, zero));
x2 = _mm256_blendv_ps(x2, zero, _mm256_cmp_ps::<_CMP_EQ_OQ>(a2, zero));
(x0, x1, x2)
}
#[inline]
#[target_feature(enable = "avx2,fma")]
pub(super) fn rgb_to_xyb_f32x8_avx2<const SCALAR_ORDER: bool>(
m: &XybMatrix,
r: __m256,
g: __m256,
b: __m256,
) -> (__m256, __m256, __m256) {
rgb_to_xyb_avx2::<SCALAR_ORDER, true>(m, r, g, b)
}
#[inline]
#[target_feature(enable = "avx2,fma")]
fn rgb_to_xyb_avx2<const SCALAR_ORDER: bool, const FUSED: bool>(
m: &XybMatrix,
r: __m256,
g: __m256,
b: __m256,
) -> (__m256, __m256, __m256) {
let bias = _mm256_set1_ps(OPSIN_BIAS);
let zero = _mm256_setzero_ps();
let row = |offset: usize| {
let mixed = mul_add_avx2::<FUSED>(b, _mm256_set1_ps(m.fwd[offset + 2]), bias);
let mixed = mul_add_avx2::<FUSED>(g, _mm256_set1_ps(m.fwd[offset + 1]), mixed);
let mixed = mul_add_avx2::<FUSED>(r, _mm256_set1_ps(m.fwd[offset]), mixed);
_mm256_max_ps(mixed, zero)
};
let (tm0, tm1, tm2) = cbrt3_positive_avx2::<SCALAR_ORDER, FUSED>(row(0), row(3), row(6));
let neg_bias = _mm256_set1_ps(NEG_BIAS_CBRT);
let tm0 = _mm256_add_ps(tm0, neg_bias);
let tm1 = _mm256_add_ps(tm1, neg_bias);
let tm2 = _mm256_add_ps(tm2, neg_bias);
let half = _mm256_set1_ps(0.5);
let x = _mm256_mul_ps(_mm256_sub_ps(tm0, tm1), half);
let y = _mm256_mul_ps(_mm256_add_ps(tm0, tm1), half);
(x, y, tm2)
}
#[inline]
#[target_feature(enable = "avx2")]
fn tail_mask_avx2(len: usize) -> __m256i {
debug_assert!(len < 8);
_mm256_cmpgt_epi32(
_mm256_set1_epi32(len as i32),
_mm256_setr_epi32(0, 1, 2, 3, 4, 5, 6, 7),
)
}
#[inline]
#[target_feature(enable = "avx2,fma")]
fn xyb_lattice_vec<const FUSED: bool>(
m: &XybMatrix,
[r, g, b]: [__m256; 3],
[sx, sy, sb]: [__m256; 3],
) -> [__m256i; 3] {
use super::quantize_xyb::round_ties_away_i32;
use crate::lossless::rct_simd::avx::forward_vec;
let (x, y, b) = rgb_to_xyb_avx2::<true, FUSED>(m, r, g, b);
let y = round_ties_away_i32(_mm256_mul_ps(y, sy));
let x = round_ties_away_i32(_mm256_mul_ps(x, sx));
let b_minus_y = _mm256_sub_epi32(round_ties_away_i32(_mm256_mul_ps(b, sb)), y);
forward_vec(y, x, b_minus_y)
}
#[target_feature(enable = "avx2,fma")]
pub(crate) unsafe fn xyb_lattice_avx2(
m: &XybMatrix,
input: [&[f32]; 3],
output: [&mut [i32]; 3],
scale: u32,
) -> usize {
if scale == 1 || cfg!(target_feature = "fma") {
unsafe { xyb_lattice_avx2_impl::<true>(m, input, output, scale) }
} else {
unsafe { xyb_lattice_avx2_impl::<false>(m, input, output, scale) }
}
}
#[inline]
#[target_feature(enable = "avx2,fma")]
unsafe fn xyb_lattice_avx2_impl<const FUSED: bool>(
m: &XybMatrix,
input: [&[f32]; 3],
output: [&mut [i32]; 3],
scale: u32,
) -> usize {
let scales = scaled_lattice_quantizers(scale).map(|quantizer| _mm256_set1_ps(quantizer));
let len = input[0].len();
let end = len / 8 * 8;
let output = output.map(|p| p.as_mut_ptr());
for i in (0..end).step_by(8) {
let rgb = input.map(|p| unsafe { _mm256_loadu_ps(p.as_ptr().add(i)) });
let values = xyb_lattice_vec::<FUSED>(m, rgb, scales);
for (p, value) in output.into_iter().zip(values) {
unsafe { _mm256_storeu_si256(p.add(i).cast(), value) };
}
}
if end < len {
let mask = tail_mask_avx2(len - end);
let rgb = input.map(|p| unsafe { _mm256_maskload_ps(p.as_ptr().add(end), mask) });
let values = xyb_lattice_vec::<FUSED>(m, rgb, scales);
for (p, value) in output.into_iter().zip(values) {
unsafe { _mm256_maskstore_epi32(p.add(end), mask, value) };
}
}
len
}
#[target_feature(enable = "avx2,fma")]
pub(crate) fn to_xyb_avx2_band(
m: &XybMatrix,
input: [&[f32]; 3],
output: [&mut [f32]; 3],
w: usize,
) {
let [rp, gp, bp] = input;
let [xp, yp, out_bp] = output;
for (((((r_row, g_row), b_row), x_row), y_row), out_b_row) in rp
.chunks_exact(w)
.zip(gp.chunks_exact(w))
.zip(bp.chunks_exact(w))
.zip(xp.chunks_exact_mut(w))
.zip(yp.chunks_exact_mut(w))
.zip(out_bp.chunks_exact_mut(w))
{
let (r_chunks, r_tail) = r_row.as_chunks::<8>();
let (g_chunks, g_tail) = g_row.as_chunks::<8>();
let (b_chunks, b_tail) = b_row.as_chunks::<8>();
let (x_chunks, x_tail) = x_row.as_chunks_mut::<8>();
let (y_chunks, y_tail) = y_row.as_chunks_mut::<8>();
let (out_b_chunks, out_b_tail) = out_b_row.as_chunks_mut::<8>();
for (((((r8, g8), b8), x8), y8), out_b8) in r_chunks
.iter()
.zip(g_chunks.iter())
.zip(b_chunks.iter())
.zip(x_chunks.iter_mut())
.zip(y_chunks.iter_mut())
.zip(out_b_chunks.iter_mut())
{
let r = unsafe { _mm256_loadu_ps(r8.as_ptr()) };
let g = unsafe { _mm256_loadu_ps(g8.as_ptr()) };
let b = unsafe { _mm256_loadu_ps(b8.as_ptr()) };
let (xv, yv, bv) = rgb_to_xyb_f32x8_avx2::<false>(m, r, g, b);
unsafe {
_mm256_storeu_ps(x8.as_mut_ptr(), xv);
_mm256_storeu_ps(y8.as_mut_ptr(), yv);
_mm256_storeu_ps(out_b8.as_mut_ptr(), bv);
}
}
if !r_tail.is_empty() {
let mask = tail_mask_avx2(r_tail.len());
let r = unsafe { _mm256_maskload_ps(r_tail.as_ptr(), mask) };
let g = unsafe { _mm256_maskload_ps(g_tail.as_ptr(), mask) };
let b = unsafe { _mm256_maskload_ps(b_tail.as_ptr(), mask) };
let (xv, yv, bv) = rgb_to_xyb_f32x8_avx2::<false>(m, r, g, b);
unsafe {
_mm256_maskstore_ps(x_tail.as_mut_ptr(), mask, xv);
_mm256_maskstore_ps(y_tail.as_mut_ptr(), mask, yv);
_mm256_maskstore_ps(out_b_tail.as_mut_ptr(), mask, bv);
}
}
}
}
#[cfg(test)]
mod tests {
use super::to_xyb_avx2_band;
use crate::xyb::{XybMatrix, rgb_to_xyb_pixel_f32};
#[test]
fn scalar_order_conversion_matches_scalar() {
if !std::is_x86_feature_detected!("avx2") || !std::is_x86_feature_detected!("fma") {
return;
}
use super::rgb_to_xyb_f32x8_avx2;
use std::arch::x86_64::{_mm256_loadu_ps, _mm256_storeu_ps};
let mut state = 0x8476_12b9u32;
let m = XybMatrix::SPEC;
for scale in [0.0, 1.0, 16.0] {
for _ in 0..1024 {
let rgb: [[f32; 8]; 3] = std::array::from_fn(|_| {
std::array::from_fn(|_| {
state ^= state << 13;
state ^= state >> 17;
state ^= state << 5;
((state >> 8) as f32 / 16777216.0 - 0.1) * scale
})
});
let mut actual = [[0.0; 8]; 3];
unsafe {
let (x, y, b) = rgb_to_xyb_f32x8_avx2::<true>(
&m,
_mm256_loadu_ps(rgb[0].as_ptr()),
_mm256_loadu_ps(rgb[1].as_ptr()),
_mm256_loadu_ps(rgb[2].as_ptr()),
);
for (dst, v) in actual.iter_mut().zip([x, y, b]) {
_mm256_storeu_ps(dst.as_mut_ptr(), v);
}
}
for i in 0..8 {
let (x, y, b) = rgb_to_xyb_pixel_f32(&m, rgb[0][i], rgb[1][i], rgb[2][i]);
if cfg!(target_feature = "fma") {
assert_eq!(actual.map(|c| c[i].to_bits()), [x, y, b].map(f32::to_bits));
} else {
for (got, expected) in actual.map(|c| c[i]).into_iter().zip([x, y, b]) {
assert!((got - expected).abs() <= 2e-6, "{got} vs {expected}");
}
}
}
}
}
}
#[test]
fn masked_tail_matches_scalar_and_preserves_canaries() {
if !std::is_x86_feature_detected!("avx2") || !std::is_x86_feature_detected!("fma") {
return;
}
const CANARY: f32 = -12_345.0;
let m = &XybMatrix::SPEC;
for w in 1..16 {
let r: Vec<_> = (0..w).map(|i| (i as f32 + 1.0) / 17.0).collect();
let g: Vec<_> = (0..w).map(|i| (2 * i + 3) as f32 / 37.0).collect();
let b: Vec<_> = (0..w).map(|i| (3 * i + 2) as f32 / 53.0).collect();
let mut x = vec![CANARY; w + 8];
let mut y = vec![CANARY; w + 8];
let mut out_b = vec![CANARY; w + 8];
unsafe {
to_xyb_avx2_band(
m,
[&r, &g, &b],
[&mut x[..w], &mut y[..w], &mut out_b[..w]],
w,
);
}
for i in 0..w {
let want = rgb_to_xyb_pixel_f32(m, r[i], g[i], b[i]);
for (got, want, channel) in [
(x[i], want.0, "X"),
(y[i], want.1, "Y"),
(out_b[i], want.2, "B"),
] {
assert!(
(got - want).abs() <= 2e-6,
"{channel} mismatch at width {w}, lane {i}: AVX {got}, scalar {want}"
);
}
}
assert!(x[w..].iter().all(|&v| v == CANARY));
assert!(y[w..].iter().all(|&v| v == CANARY));
assert!(out_b[w..].iter().all(|&v| v == CANARY));
}
}
}