#[cfg(target_arch = "x86")]
use std::arch::x86::*;
#[cfg(target_arch = "x86_64")]
use std::arch::x86_64::*;
#[target_feature(enable = "sse4.1")]
unsafe fn iload(p: *const i32) -> __m128i {
unsafe { _mm_loadu_si128(p.cast()) }
}
#[target_feature(enable = "sse4.1")]
unsafe fn istore(p: *mut i32, v: __m128i) {
unsafe { _mm_storeu_si128(p.cast(), v) }
}
#[target_feature(enable = "sse4.1")]
fn shr<const N: i32>(v: __m128i) -> __m128i {
_mm_srai_epi32::<N>(v)
}
#[target_feature(enable = "sse4.1")]
pub(crate) fn rct_row_sse41(src: [&[i32]; 3], kind: u32, out: [&mut [i32]; 3]) {
let [a, b, c] = out;
let len = a.len();
assert!(src.iter().all(|s| s.len() >= len) && b.len() >= len && c.len() >= len);
assert!(kind < 42);
if kind == 6 {
for (out, src) in [a, b, c].into_iter().zip(src) {
out.copy_from_slice(src);
}
return;
}
let perm = (kind / 7) as usize;
let perm = [
perm % 3,
(perm + 1 + perm / 3) % 3,
(perm + 2 - perm / 3) % 3,
];
let t = kind % 7;
let end = len / 4 * 4;
for i in (0..end).step_by(4) {
let [y, co, cg] = src.map(|s| unsafe { iload(s.as_ptr().add(i)) });
let tmp = _mm_sub_epi32(y, shr::<1>(cg));
let blue = _mm_sub_epi32(tmp, shr::<1>(co));
let rgb = [_mm_add_epi32(blue, co), _mm_add_epi32(cg, tmp), blue];
let [first, second, third] = perm.map(|p| rgb[p]);
let (v0, v1, v2) = if t == 6 {
let [y, co, cg] = crate::lossless::rct_simd::sse::forward_vec(first, second, third);
(y, co, cg)
} else {
let v1 = match t >> 1 {
1 => _mm_sub_epi32(second, first),
2 => _mm_sub_epi32(second, shr::<1>(_mm_add_epi32(first, third))),
_ => second,
};
(
first,
v1,
if t & 1 == 1 {
_mm_sub_epi32(third, first)
} else {
third
},
)
};
unsafe {
istore(a.as_mut_ptr().add(i), v0);
istore(b.as_mut_ptr().add(i), v1);
istore(c.as_mut_ptr().add(i), v2);
}
}
crate::lossless::rct_row_scalar(
src.map(|s| &s[end..]),
kind,
[&mut a[end..], &mut b[end..], &mut c[end..]],
);
}
#[target_feature(enable = "sse4.1")]
pub(crate) fn palette_row_sse41(above: &[i32], row: &[i32], out: &mut [[i32; 4]]) {
assert!(above.len() >= out.len() + 3 && row.len() >= out.len() + 2);
let end = out.len() / 4 * 4;
for x in (0..end).step_by(4) {
let [ll, l, v] = [0, 1, 2].map(|j| unsafe { iload(row.as_ptr().add(x + j)) });
let [tl, t, tr] = [1, 2, 3].map(|j| unsafe { iload(above.as_ptr().add(x + j)) });
let sign = |v| {
_mm_add_epi32(
_mm_sub_epi32(_mm_set1_epi32(1), _mm_cmpgt_epi32(v, _mm_set1_epi32(0))),
_mm_cmpgt_epi32(_mm_set1_epi32(0), v),
)
};
let ctx = _mm_add_epi32(
_mm_add_epi32(
_mm_mullo_epi32(sign(_mm_sub_epi32(l, tl)), _mm_set1_epi32(27)),
_mm_mullo_epi32(sign(_mm_sub_epi32(t, tl)), _mm_set1_epi32(9)),
),
_mm_add_epi32(
_mm_mullo_epi32(sign(_mm_sub_epi32(t, tr)), _mm_set1_epi32(3)),
sign(_mm_sub_epi32(l, ll)),
),
);
let gradient = _mm_min_epi32(
_mm_max_epi32(_mm_sub_epi32(_mm_add_epi32(l, t), tl), _mm_min_epi32(l, t)),
_mm_max_epi32(l, t),
);
let vectors = [
ctx,
_mm_sub_epi32(v, l),
_mm_sub_epi32(v, t),
_mm_sub_epi32(v, gradient),
];
let mut values = [[0; 4]; 4];
for (dst, vector) in values.iter_mut().zip(vectors) {
unsafe {
istore(dst.as_mut_ptr(), vector);
}
}
for i in 0..4 {
out[x + i] = std::array::from_fn(|j| values[j][i]);
}
}
crate::lossless::palette_row_scalar(&above[end..], &row[end..], &mut out[end..]);
}