#[cfg(target_arch = "x86")]
use std::arch::x86::*;
#[cfg(target_arch = "x86_64")]
use std::arch::x86_64::*;
#[target_feature(enable = "sse4.1")]
unsafe fn iload(p: *const i32) -> __m128i {
unsafe { _mm_loadu_si128(p.cast()) }
}
#[target_feature(enable = "sse4.1")]
unsafe fn istore(p: *mut i32, v: __m128i) {
unsafe { _mm_storeu_si128(p.cast(), v) }
}
#[target_feature(enable = "sse4.1")]
fn any(v: __m128i) -> bool {
_mm_movemask_epi8(v) != 0
}
#[target_feature(enable = "sse4.1")]
pub(crate) fn luma_row_sse41(rgb: &[[u8; 3]], out: &mut [u8]) {
assert!(rgb.len() >= out.len());
let end = out.len() / 16 * 16;
for (x, pixels) in rgb[..end].as_chunks::<16>().0.iter().enumerate() {
let x = x * 16;
let channels = crate::simd::sse::deinterleave_rgb8(pixels);
let zero = _mm_setzero_si128();
let half = |hi: bool| {
let [r, g, b] = channels.map(|v| {
if hi {
_mm_unpackhi_epi8(v, zero)
} else {
_mm_unpacklo_epi8(v, zero)
}
});
let y = _mm_add_epi16(
_mm_add_epi16(
_mm_mullo_epi16(r, _mm_set1_epi16(77)),
_mm_mullo_epi16(g, _mm_set1_epi16(150)),
),
_mm_add_epi16(_mm_mullo_epi16(b, _mm_set1_epi16(29)), _mm_set1_epi16(128)),
);
_mm_srli_epi16::<8>(y)
};
let y = _mm_packus_epi16(half(false), half(true));
unsafe {
_mm_storeu_si128(out.as_mut_ptr().add(x).cast(), y);
}
}
crate::ink_layer::luma_row_scalar(&rgb[end..], &mut out[end..]);
}
#[target_feature(enable = "sse4.1")]
pub(crate) fn ridge_row_sse41(
rgb: &[[u8; 3]],
luma: &[u8],
w: usize,
h: usize,
y: usize,
out: &mut [u8],
) -> usize {
assert!(rgb.len() >= w * h && luma.len() >= w * h && out.len() >= w && y < h);
let mut x = y & 1;
let mut count = 0;
while x + 2 * (4 - 1) < w {
let positions: [[usize; 5]; 4] =
std::array::from_fn(|i| crate::ink_layer::ridge_positions(x + 2 * i, y, w, h));
let gather = |p: usize, c: Option<usize>| {
let v: [i32; 4] = std::array::from_fn(|i| {
let j = positions[i][p];
i32::from(c.map_or(luma[j], |c| rgb[j][c]))
});
unsafe { iload(v.as_ptr()) }
};
let mut yes = _mm_set1_epi32(0);
let p = gather(0, None);
for (a, b) in [(1, 2), (3, 4)] {
let da = _mm_sub_epi32(p, gather(a, None));
let db = _mm_sub_epi32(p, gather(b, None));
yes = _mm_or_si128(
yes,
_mm_or_si128(
_mm_and_si128(
_mm_cmpgt_epi32(da, _mm_set1_epi32(crate::ink_layer::THIN_DEV)),
_mm_cmpgt_epi32(db, _mm_set1_epi32(crate::ink_layer::THIN_DEV)),
),
_mm_and_si128(
_mm_cmpgt_epi32(_mm_set1_epi32(-crate::ink_layer::THIN_DEV), da),
_mm_cmpgt_epi32(_mm_set1_epi32(-crate::ink_layer::THIN_DEV), db),
),
),
);
}
if any(_mm_sub_epi32(_mm_set1_epi32(-1), yes)) {
let center: [_; 3] = std::array::from_fn(|c| gather(0, Some(c)));
let chroma = |d: [_; 3]| {
_mm_max_epi32(
_mm_abs_epi32(_mm_sub_epi32(d[0], d[1])),
_mm_abs_epi32(_mm_sub_epi32(d[2], d[1])),
)
};
for (a, b) in [(1, 2), (3, 4)] {
let ac: [_; 3] =
std::array::from_fn(|c| _mm_sub_epi32(center[c], gather(a, Some(c))));
let possible = _mm_and_si128(
_mm_cmpgt_epi32(chroma(ac), _mm_set1_epi32(crate::ink_layer::THIN_DEV)),
_mm_sub_epi32(_mm_set1_epi32(-1), yes),
);
if any(possible) {
let bc: [_; 3] =
std::array::from_fn(|c| _mm_sub_epi32(center[c], gather(b, Some(c))));
let mut background = _mm_set1_epi32(-1);
for c in 0..3 {
background = _mm_and_si128(
background,
_mm_cmpgt_epi32(
_mm_set1_epi32(16),
_mm_abs_epi32(_mm_sub_epi32(bc[c], ac[c])),
),
);
}
yes = _mm_or_si128(
yes,
_mm_and_si128(
possible,
_mm_and_si128(
background,
_mm_cmpgt_epi32(
chroma(bc),
_mm_set1_epi32(crate::ink_layer::THIN_DEV),
),
),
),
);
}
}
}
let mut values = [0; 4];
unsafe {
istore(values.as_mut_ptr(), yes);
}
for i in 0..4 {
out[x + 2 * i] = u8::from(values[i] != 0);
count += usize::from(out[x + 2 * i]);
}
x += 2 * 4;
}
for x in (x..w).step_by(2) {
out[x] =
crate::ink_layer::ridge_pixel(rgb, luma, crate::ink_layer::ridge_positions(x, y, w, h));
count += usize::from(out[x]);
}
count
}