use super::forward_ycocg;
use std::sync::OnceLock;
type ForwardPlanes = unsafe fn([&[i32]; 3], [&mut [i32]; 3]);
type ForwardInPlace = unsafe fn([&mut [i32]; 3]);
type ForwardRgb8 = unsafe fn(&[[u8; 3]], [&mut [i32]; 3]);
struct Kernels {
planes: ForwardPlanes,
in_place: ForwardInPlace,
rgb8: ForwardRgb8,
}
fn selected() -> &'static Kernels {
static KERNELS: OnceLock<Kernels> = OnceLock::new();
KERNELS.get_or_init(select)
}
fn select() -> Kernels {
#[cfg(all(target_arch = "x86_64", feature = "avx"))]
if std::is_x86_feature_detected!("avx2") {
return avx::kernels();
}
#[cfg(all(any(target_arch = "x86", target_arch = "x86_64"), feature = "sse"))]
if std::is_x86_feature_detected!("sse4.1") {
return sse::kernels();
}
#[cfg(all(target_arch = "aarch64", feature = "neon"))]
return neon::kernels();
#[cfg(all(target_arch = "wasm32", feature = "wasm", target_feature = "simd128"))]
return wasm::kernels();
#[cfg(not(any(
all(target_arch = "aarch64", feature = "neon"),
all(target_arch = "wasm32", feature = "wasm", target_feature = "simd128")
)))]
Kernels {
planes: planes_scalar,
in_place: in_place_scalar,
rgb8: rgb8_scalar,
}
}
pub(crate) fn forward_ycocg_planes(src: [&[i32]; 3], dst: [&mut [i32]; 3]) {
assert!(src.iter().all(|p| p.len() == dst[0].len()));
assert!(dst.iter().all(|p| p.len() == src[0].len()));
unsafe { (selected().planes)(src, dst) };
}
pub(crate) fn forward_ycocg_in_place(planes: [&mut [i32]; 3]) {
assert!(planes.iter().all(|p| p.len() == planes[0].len()));
unsafe { (selected().in_place)(planes) };
}
pub(crate) fn rgb8_to_ycocg(src: &[[u8; 3]], dst: [&mut [i32]; 3]) {
assert!(dst.iter().all(|p| p.len() == src.len()));
unsafe { (selected().rgb8)(src, dst) };
}
fn planes_scalar([r, g, b]: [&[i32]; 3], [y, co, cg]: [&mut [i32]; 3]) {
let src = r.iter().zip(g).zip(b);
for (((&r, &g), &b), ((y, co), cg)) in src.zip(y.iter_mut().zip(co).zip(cg)) {
(*y, *co, *cg) = forward_ycocg(r, g, b);
}
}
fn in_place_scalar([r, g, b]: [&mut [i32]; 3]) {
for ((r, g), b) in r.iter_mut().zip(g).zip(b) {
(*r, *g, *b) = forward_ycocg(*r, *g, *b);
}
}
fn rgb8_scalar(src: &[[u8; 3]], [y, co, cg]: [&mut [i32]; 3]) {
for (&pixel, ((y, co), cg)) in src.iter().zip(y.iter_mut().zip(co).zip(cg)) {
let [r, g, b] = pixel.map(i32::from);
(*y, *co, *cg) = forward_ycocg(r, g, b);
}
}
#[cfg(any(
all(target_arch = "x86_64", feature = "avx"),
all(any(target_arch = "x86", target_arch = "x86_64"), feature = "sse"),
all(target_arch = "aarch64", feature = "neon"),
all(target_arch = "wasm32", feature = "wasm", target_feature = "simd128")
))]
macro_rules! backend {
($feature:literal, $lanes:literal, [$($offset:literal),+]) => {
pub(super) fn kernels() -> super::Kernels {
super::Kernels { planes, in_place, rgb8 }
}
#[inline]
#[target_feature(enable = $feature)]
pub(crate) fn forward_vec(r: Vector, g: Vector, b: Vector) -> [Vector; 3] {
let co = sub(r, b);
let tmp = add(b, shr::<1>(co));
let cg = sub(g, tmp);
[add(tmp, shr::<1>(cg)), co, cg]
}
#[target_feature(enable = $feature)]
unsafe fn planes(src: [&[i32]; 3], dst: [&mut [i32]; 3]) {
let [y, co, cg] = dst;
let end = y.len() / $lanes * $lanes;
for i in (0..end).step_by($lanes) {
let [r, g, b] = src.map(|p| unsafe { load(p.as_ptr().add(i)) });
let [luma, orange, green] = forward_vec(r, g, b);
unsafe {
store(y.as_mut_ptr().add(i), luma);
store(co.as_mut_ptr().add(i), orange);
store(cg.as_mut_ptr().add(i), green);
}
}
super::planes_scalar(src.map(|p| &p[end..]), [&mut y[end..], &mut co[end..], &mut cg[end..]]);
}
#[target_feature(enable = $feature)]
unsafe fn in_place([r, g, b]: [&mut [i32]; 3]) {
let end = r.len() / $lanes * $lanes;
for i in (0..end).step_by($lanes) {
let ptrs = [&mut *r, &mut *g, &mut *b].map(|p| unsafe { p.as_mut_ptr().add(i) });
let [rv, gv, bv] = ptrs.map(|p| unsafe { load(p) });
for (p, value) in ptrs.into_iter().zip(forward_vec(rv, gv, bv)) {
unsafe { store(p, value) };
}
}
super::in_place_scalar([&mut r[end..], &mut g[end..], &mut b[end..]]);
}
#[target_feature(enable = $feature)]
unsafe fn rgb8(src: &[[u8; 3]], [y, co, cg]: [&mut [i32]; 3]) {
let (blocks, tail) = src.as_chunks::<16>();
let end = blocks.len() * 16;
for (block, pixels) in blocks.iter().enumerate() {
let channels = deinterleave(pixels);
let i = block * 16;
$({
let [r, g, b] = channels.map(|channel| widen::<$offset>(channel));
let [luma, orange, green] = forward_vec(r, g, b);
unsafe {
store(y.as_mut_ptr().add(i + $offset), luma);
store(co.as_mut_ptr().add(i + $offset), orange);
store(cg.as_mut_ptr().add(i + $offset), green);
}
})+
}
super::rgb8_scalar(tail, [&mut y[end..], &mut co[end..], &mut cg[end..]]);
}
};
}
#[cfg(all(target_arch = "aarch64", feature = "neon"))]
pub(crate) mod neon {
use crate::simd::neon::i32::*;
use std::arch::aarch64::*;
backend!("neon", 4, [0, 4, 8, 12]);
#[inline]
#[target_feature(enable = "neon")]
fn deinterleave(pixels: &[[u8; 3]; 16]) -> [uint8x16_t; 3] {
let uint8x16x3_t(r, g, b) = unsafe { vld3q_u8(pixels.as_ptr().cast()) };
[r, g, b]
}
#[inline]
#[target_feature(enable = "neon")]
fn widen<const LANE: usize>(channel: uint8x16_t) -> Vector {
let half = if LANE < 8 {
vmovl_u8(vget_low_u8(channel))
} else {
vmovl_high_u8(channel)
};
let words = if LANE.is_multiple_of(8) {
vmovl_u16(vget_low_u16(half))
} else {
vmovl_high_u16(half)
};
vreinterpretq_s32_u32(words)
}
}
#[cfg(all(target_arch = "x86_64", feature = "avx"))]
pub(crate) mod avx {
use crate::simd::avx::i32::*;
use crate::simd::sse::deinterleave_rgb8 as deinterleave;
use std::arch::x86_64::*;
backend!("avx2", 8, [0, 8]);
#[inline]
#[target_feature(enable = "avx2")]
fn widen<const LANE: usize>(channel: __m128i) -> Vector {
let bytes = if LANE == 0 {
channel
} else {
_mm_srli_si128::<8>(channel)
};
_mm256_cvtepu8_epi32(bytes)
}
}
#[cfg(all(any(target_arch = "x86", target_arch = "x86_64"), feature = "sse"))]
pub(crate) mod sse {
use crate::simd::sse::deinterleave_rgb8 as deinterleave;
use crate::simd::sse::i32::*;
#[cfg(target_arch = "x86")]
use std::arch::x86::*;
#[cfg(target_arch = "x86_64")]
use std::arch::x86_64::*;
backend!("sse4.1", 4, [0, 4, 8, 12]);
#[inline]
#[target_feature(enable = "sse4.1")]
fn widen<const LANE: usize>(channel: __m128i) -> Vector {
let bytes = match LANE {
0 => channel,
4 => _mm_srli_si128::<4>(channel),
8 => _mm_srli_si128::<8>(channel),
_ => _mm_srli_si128::<12>(channel),
};
_mm_cvtepu8_epi32(bytes)
}
}
#[cfg(all(target_arch = "wasm32", feature = "wasm", target_feature = "simd128"))]
pub(crate) mod wasm {
use crate::simd::wasm::deinterleave_rgb8 as deinterleave;
use crate::simd::wasm::i32::*;
use std::arch::wasm32::*;
backend!("simd128", 4, [0, 4, 8, 12]);
#[inline]
#[target_feature(enable = "simd128")]
fn widen<const LANE: usize>(channel: v128) -> Vector {
let half = if LANE < 8 {
u16x8_extend_low_u8x16(channel)
} else {
u16x8_extend_high_u8x16(channel)
};
if LANE.is_multiple_of(8) {
u32x4_extend_low_u16x8(half)
} else {
u32x4_extend_high_u16x8(half)
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn packed_and_planar_ycocg_match_scalar_with_tails() {
let bytes: Vec<_> = (0..140)
.map(|i| [(i * 73) as u8, (i * 137 + 19) as u8, (i * 53 + 251) as u8])
.collect();
let signed: [Vec<_>; 3] = std::array::from_fn(|c| {
(0..bytes.len())
.map(|i| i32::from(bytes[i][c]) * 511 - 65_535)
.collect()
});
const CANARY: i32 = i32::MIN;
for len in 0..=bytes.len() - 2 {
let range = 1..1 + len;
let mut packed = std::array::from_fn(|_| vec![CANARY; len + 2]);
rgb8_to_ycocg(
&bytes[range.clone()],
packed.each_mut().map(|p| &mut p[range.clone()]),
);
let src = signed.each_ref().map(|p| &p[range.clone()]);
let mut planar = std::array::from_fn(|_| vec![CANARY; len + 2]);
forward_ycocg_planes(src, planar.each_mut().map(|p| &mut p[range.clone()]));
let mut in_place = signed.clone();
forward_ycocg_in_place(in_place.each_mut().map(|p| &mut p[range.clone()]));
for i in range {
let [r, g, b] = bytes[i].map(i32::from);
let (y, co, cg) = forward_ycocg(r, g, b);
assert_eq!(packed.each_ref().map(|p| p[i]), [y, co, cg]);
let [r, g, b] = signed.each_ref().map(|p| p[i]);
let (y, co, cg) = forward_ycocg(r, g, b);
assert_eq!(planar.each_ref().map(|p| p[i]), [y, co, cg]);
assert_eq!(in_place.each_ref().map(|p| p[i]), [y, co, cg]);
}
for p in packed.iter().chain(&planar) {
assert_eq!((p[0], p[len + 1]), (CANARY, CANARY));
}
for (p, original) in in_place.iter().zip(&signed) {
assert_eq!((p[0], p[len + 1]), (original[0], original[len + 1]));
}
}
}
}