use core::arch::wasm32::*;
#[inline]
#[target_feature(enable = "simd128")]
fn accumulate_gradient_vectors_x4(left: v128, right: v128, sum: v128) -> v128 {
let difference = f32x4_sub(right, left);
f32x4_add(sum, f32x4_mul(difference, difference))
}
#[inline]
#[target_feature(enable = "simd128")]
fn accumulate_gradient_x4(left: &[f32; 4], right: &[f32; 4], sum: v128) -> v128 {
let left = unsafe { v128_load(left.as_ptr().cast()) };
let right = unsafe { v128_load(right.as_ptr().cast()) };
accumulate_gradient_vectors_x4(left, right, sum)
}
#[target_feature(enable = "simd128")]
pub(crate) fn error_gradient_energy_wasm(error: &[f32], width: usize, height: usize) -> f32 {
let n = width
.checked_mul(height)
.expect("gradient plane size overflow");
assert!(error.len() >= n);
if width == 0 || height == 0 {
return 0.0;
}
let rows = error[..n].chunks_exact(width);
let mut sum = f32x4_splat(0.0);
for row in rows.clone() {
let (row4, row_tail) = row.as_chunks::<4>();
if row_tail.is_empty() {
let (right4, _) = row[1..].as_chunks::<4>();
for (left, right) in row4.iter().zip(right4) {
sum = accumulate_gradient_x4(left, right, sum);
}
let left = unsafe { v128_load(row4.last().unwrap().as_ptr().cast()) };
sum =
accumulate_gradient_vectors_x4(left, i32x4_shuffle::<1, 2, 3, 3>(left, left), sum);
continue;
}
let (left4, tail) = row[..width - 1].as_chunks::<4>();
let (right4, right_tail) = row[1..].as_chunks::<4>();
for (left, right) in left4.iter().zip(right4) {
sum = accumulate_gradient_x4(left, right, sum);
}
if !tail.is_empty() {
let mut left = [0.0; 4];
let mut right = [0.0; 4];
left[..tail.len()].copy_from_slice(tail);
right[..right_tail.len()].copy_from_slice(right_tail);
sum = accumulate_gradient_x4(&left, &right, sum);
}
}
for (top, bottom) in rows.clone().zip(rows.skip(1)) {
let (top4, top_tail) = top.as_chunks::<4>();
let (bottom4, bottom_tail) = bottom.as_chunks::<4>();
for (top, bottom) in top4.iter().zip(bottom4) {
sum = accumulate_gradient_x4(top, bottom, sum);
}
if !top_tail.is_empty() {
let mut top = [0.0; 4];
let mut bottom = [0.0; 4];
top[..top_tail.len()].copy_from_slice(top_tail);
bottom[..bottom_tail.len()].copy_from_slice(bottom_tail);
sum = accumulate_gradient_x4(&top, &bottom, sum);
}
}
f32x4_extract_lane::<0>(sum)
+ f32x4_extract_lane::<1>(sum)
+ f32x4_extract_lane::<2>(sum)
+ f32x4_extract_lane::<3>(sum)
}
#[inline]
#[target_feature(enable = "simd128")]
fn combine_error_x4(spatial: &[f32; 4], luma: &[f32; 4], factor: v128, combined: &mut [f32; 4]) {
let spatial = unsafe { v128_load(spatial.as_ptr().cast()) };
let luma = unsafe { v128_load(luma.as_ptr().cast()) };
let value = f32x4_add(f32x4_mul(factor, luma), spatial);
unsafe { v128_store(combined.as_mut_ptr().cast(), value) };
}
#[target_feature(enable = "simd128")]
pub(crate) fn combine_error_wasm(spatial: &[f32], luma: &[f32], factor: f32, combined: &mut [f32]) {
debug_assert_eq!(spatial.len(), luma.len());
debug_assert_eq!(spatial.len(), combined.len());
let (spatial16, spatial_tail) = spatial.as_chunks::<16>();
let (luma16, luma_tail) = luma.as_chunks::<16>();
let (combined16, combined_tail) = combined.as_chunks_mut::<16>();
let factor = f32x4_splat(factor);
for ((spatial, luma), combined) in spatial16.iter().zip(luma16).zip(combined16) {
let [s0, s1, s2, s3] = spatial.as_chunks::<4>().0 else {
unreachable!()
};
let [l0, l1, l2, l3] = luma.as_chunks::<4>().0 else {
unreachable!()
};
let [c0, c1, c2, c3] = combined.as_chunks_mut::<4>().0 else {
unreachable!()
};
combine_error_x4(s0, l0, factor, c0);
combine_error_x4(s1, l1, factor, c1);
combine_error_x4(s2, l2, factor, c2);
combine_error_x4(s3, l3, factor, c3);
}
let (spatial4, spatial_remainder) = spatial_tail.as_chunks::<4>();
let (luma4, luma_remainder) = luma_tail.as_chunks::<4>();
let (combined4, combined_remainder) = combined_tail.as_chunks_mut::<4>();
for ((spatial, luma), combined) in spatial4.iter().zip(luma4).zip(combined4) {
combine_error_x4(spatial, luma, factor, combined);
}
if !spatial_remainder.is_empty() {
let mut spatial = [0.0; 4];
let mut luma = [0.0; 4];
let mut combined = [0.0; 4];
spatial[..spatial_remainder.len()].copy_from_slice(spatial_remainder);
luma[..luma_remainder.len()].copy_from_slice(luma_remainder);
combine_error_x4(&spatial, &luma, factor, &mut combined);
combined_remainder.copy_from_slice(&combined[..combined_remainder.len()]);
}
}