use core::arch::wasm32::*;
#[inline]
#[target_feature(enable = "simd128")]
fn hsum4(v: v128) -> f32 {
f32x4_extract_lane::<0>(v)
+ f32x4_extract_lane::<1>(v)
+ f32x4_extract_lane::<2>(v)
+ f32x4_extract_lane::<3>(v)
}
#[inline]
#[target_feature(enable = "simd128")]
pub(super) fn wasm_dequantized_level_f32(q: v128, bias: crate::group::DequantBias) -> v128 {
let absq = f32x4_abs(q);
let big = f32x4_sub(q, f32x4_div(f32x4_splat(bias.numerator), q));
let sign = v128_and(q, f32x4_splat(-0.0));
let one = v128_or(sign, f32x4_splat(bias.one));
let use_big = f32x4_ge(absq, f32x4_splat(1.125));
let dq = v128_bitselect(big, one, use_big);
let nz = f32x4_gt(absq, f32x4_splat(0.0));
v128_and(dq, nz)
}
#[allow(clippy::too_many_arguments)]
#[target_feature(enable = "simd128")]
pub(crate) fn sse_and_rate_wasm<const BIASED: bool>(
coeff: &[f32],
inv_matrix: &[f32],
q_scaled: f32,
bias: crate::group::DequantBias,
width: usize,
height: usize,
half: usize,
cx: usize,
cy: usize,
_rate_log2_lut: &crate::inflated_cost::RateLog2Lut,
thr: &[f32; 4],
scan_pos: &[u32],
) -> (f32, usize, f32, u32) {
sse_and_rate_wasm_impl::<BIASED, false>(
coeff,
inv_matrix,
q_scaled,
bias,
width,
height,
half,
cx,
cy,
_rate_log2_lut,
thr,
scan_pos,
&mut [],
)
}
#[allow(clippy::too_many_arguments)]
#[target_feature(enable = "simd128")]
pub(crate) fn sse_and_rate_wasm_impl<const BIASED: bool, const SAVE_LEVELS: bool>(
coeff: &[f32],
inv_matrix: &[f32],
q_scaled: f32,
bias: crate::group::DequantBias,
width: usize,
height: usize,
half: usize,
cx: usize,
cy: usize,
_rate_log2_lut: &crate::inflated_cost::RateLog2Lut,
thr: &[f32; 4],
scan_pos: &[u32],
levels: &mut [i32],
) -> (f32, usize, f32, u32) {
let n = width * height;
assert!(coeff.len() >= n && inv_matrix.len() >= n);
assert!(if SAVE_LEVELS {
levels.len() >= n
} else {
scan_pos.len() >= n
});
debug_assert!(width.is_multiple_of(4) && half.is_multiple_of(4));
let qs = f32x4_splat(q_scaled);
let mut sse_acc = f32x4_splat(0.0);
let mut mag_acc = f32x4_splat(0.0);
let mut nonzero_acc = u32x4_splat(0);
let mut scan_acc = u32x4_splat(0);
let zero = f32x4_splat(0.0);
let lane_ids_arr = [0i32, 1, 2, 3];
let lane_ids = unsafe { v128_load(lane_ids_arr.as_ptr() as *const v128) };
for (y, (coeffs, inv_matrix)) in coeff
.chunks_exact(width)
.zip(inv_matrix.chunks_exact(width))
.take(height)
.enumerate()
{
let yfix = if y >= height / 2 { 2 } else { 0 };
let thr_lo = thr[yfix];
let thr_hi = thr[yfix + 1];
let mut prepare = |x: usize| {
let threshold = if x >= half { thr_hi } else { thr_lo };
let cv = unsafe { v128_load(coeffs.as_ptr().add(x).cast()) };
let mv = unsafe { v128_load(inv_matrix.as_ptr().add(x).cast()) };
let a = f32x4_mul(f32x4_mul(mv, qs), cv);
let keep = f32x4_ge(f32x4_abs(a), f32x4_splat(threshold));
let q = v128_and(f32x4_nearest(a), keep);
let d = if BIASED {
f32x4_sub(a, wasm_dequantized_level_f32(q, bias))
} else {
f32x4_sub(a, q)
};
let active = if y < cy && x < cx {
i32x4_ge(
i32x4_add(i32x4_splat(x as i32), lane_ids),
i32x4_splat(cx as i32),
)
} else {
i32x4_splat(-1)
};
let d2 = v128_bitselect(f32x4_mul(d, d), zero, active);
let absq = f32x4_abs(q);
let rate_mask = v128_and(f32x4_gt(absq, zero), active);
if SAVE_LEVELS {
let truncated = f32x4_trunc(a);
let frac = f32x4_sub(a, truncated);
let ge_half = f32x4_ge(f32x4_abs(frac), f32x4_splat(0.5));
let signed_one = v128_or(f32x4_splat(1.0), v128_and(a, f32x4_splat(-0.0)));
let quantized =
i32x4_trunc_sat_f32x4(f32x4_add(truncated, v128_and(signed_one, ge_half)));
let quantized = v128_and(quantized, v128_and(keep, active));
unsafe { v128_store(levels.as_mut_ptr().add(y * width + x).cast(), quantized) };
}
(d2, absq, rate_mask)
};
let paired = width / 8 * 8;
for x in (0..paired).step_by(8) {
let (d0, q0, mask0) = prepare(x);
let (d1, q1, mask1) = prepare(x + 4);
sse_acc = f32x4_add(sse_acc, d0);
sse_acc = f32x4_add(sse_acc, d1);
nonzero_acc = i32x4_sub(nonzero_acc, mask0);
nonzero_acc = i32x4_sub(nonzero_acc, mask1);
if !SAVE_LEVELS {
let bits0 = i32x4_bitmask(mask0);
let bits1 = i32x4_bitmask(mask1);
if bits0 != 0 && bits1 != 0 {
let (r0, r1) = (wasm_log2p1_f32(q0), wasm_log2p1_f32(q1));
mag_acc = f32x4_add(mag_acc, v128_bitselect(r0, zero, mask0));
mag_acc = f32x4_add(mag_acc, v128_bitselect(r1, zero, mask1));
} else if bits0 != 0 {
mag_acc = f32x4_add(mag_acc, v128_bitselect(wasm_log2p1_f32(q0), zero, mask0));
} else if bits1 != 0 {
mag_acc = f32x4_add(mag_acc, v128_bitselect(wasm_log2p1_f32(q1), zero, mask1));
}
if bits0 != 0 {
let sv = unsafe { v128_load(scan_pos.as_ptr().add(y * width + x).cast()) };
scan_acc = u32x4_max(scan_acc, v128_and(sv, mask0));
}
if bits1 != 0 {
let sv = unsafe { v128_load(scan_pos.as_ptr().add(y * width + x + 4).cast()) };
scan_acc = u32x4_max(scan_acc, v128_and(sv, mask1));
}
}
}
if paired + 4 <= width {
let x = paired;
let (d, q, mask) = prepare(x);
sse_acc = f32x4_add(sse_acc, d);
nonzero_acc = i32x4_sub(nonzero_acc, mask);
if !SAVE_LEVELS && i32x4_bitmask(mask) != 0 {
mag_acc = f32x4_add(mag_acc, v128_bitselect(wasm_log2p1_f32(q), zero, mask));
let sv = unsafe { v128_load(scan_pos.as_ptr().add(y * width + x).cast()) };
scan_acc = u32x4_max(scan_acc, v128_and(sv, mask));
}
}
}
let max_scan = u32x4_extract_lane::<0>(scan_acc)
.max(u32x4_extract_lane::<1>(scan_acc))
.max(u32x4_extract_lane::<2>(scan_acc))
.max(u32x4_extract_lane::<3>(scan_acc));
let nzeros = (u32x4_extract_lane::<0>(nonzero_acc)
+ u32x4_extract_lane::<1>(nonzero_acc)
+ u32x4_extract_lane::<2>(nonzero_acc)
+ u32x4_extract_lane::<3>(nonzero_acc)) as usize;
(hsum4(sse_acc), nzeros, hsum4(mag_acc), max_scan)
}
#[cfg(test)]
#[allow(clippy::too_many_arguments)]
#[target_feature(enable = "simd128")]
pub(crate) fn sse_and_rate_wasm_impl_reference<const BIASED: bool, const SAVE_LEVELS: bool>(
coeff: &[f32],
inv_matrix: &[f32],
q_scaled: f32,
bias: crate::group::DequantBias,
width: usize,
height: usize,
half: usize,
cx: usize,
cy: usize,
_rate_log2_lut: &crate::inflated_cost::RateLog2Lut,
thr: &[f32; 4],
scan_pos: &[u32],
levels: &mut [i32],
) -> (f32, usize, f32, u32) {
let n = width * height;
assert!(coeff.len() >= n && inv_matrix.len() >= n);
assert!(if SAVE_LEVELS {
levels.len() >= n
} else {
scan_pos.len() >= n
});
debug_assert!(width.is_multiple_of(4) && half.is_multiple_of(4));
let qs = f32x4_splat(q_scaled);
let mut sse_acc = f32x4_splat(0.0);
let mut mag_acc = f32x4_splat(0.0);
let mut nzeros = 0usize;
let mut scan_acc = u32x4_splat(0);
let zero = f32x4_splat(0.0);
let lane_ids_arr = [0i32, 1, 2, 3];
let lane_ids = unsafe { v128_load(lane_ids_arr.as_ptr() as *const v128) };
for (y, (coeffs, inv_matrix)) in coeff
.chunks_exact(width)
.zip(inv_matrix.chunks_exact(width))
.take(height)
.enumerate()
{
let yfix = if y >= height / 2 { 2 } else { 0 };
let thr_lo = thr[yfix];
let thr_hi = thr[yfix + 1];
for (x0, (coeff4, inv4)) in coeffs
.as_chunks::<4>()
.0
.iter()
.zip(inv_matrix.as_chunks::<4>().0.iter())
.enumerate()
{
let x = x0 * 4;
let threshold = if x >= half { thr_hi } else { thr_lo };
let thrv = f32x4_splat(threshold);
let cv = unsafe { v128_load(coeff4.as_ptr() as *const v128) };
let mv = unsafe { v128_load(inv4.as_ptr() as *const v128) };
let a = f32x4_mul(f32x4_mul(mv, qs), cv);
let absa = f32x4_abs(a);
let keep = f32x4_ge(absa, thrv);
let rounded = f32x4_nearest(a);
let q = v128_and(rounded, keep);
let d = if BIASED {
f32x4_sub(a, wasm_dequantized_level_f32(q, bias))
} else {
f32x4_sub(a, q)
};
let d2 = f32x4_mul(d, d);
let active = if y < cy && x < cx {
let lane_x = i32x4_add(i32x4_splat(x as i32), lane_ids);
i32x4_ge(lane_x, i32x4_splat(cx as i32))
} else {
i32x4_splat(-1)
};
let d2 = v128_bitselect(d2, zero, active);
sse_acc = f32x4_add(sse_acc, d2);
let absq = f32x4_abs(q);
let nz = f32x4_gt(absq, zero);
let rate_mask = v128_and(nz, active);
let rate_bits = i32x4_bitmask(rate_mask);
nzeros += rate_bits.count_ones() as usize;
if SAVE_LEVELS {
let truncated = f32x4_trunc(a);
let frac = f32x4_sub(a, truncated);
let ge_half = f32x4_ge(f32x4_abs(frac), f32x4_splat(0.5));
let signed_one = v128_or(f32x4_splat(1.0), v128_and(a, f32x4_splat(-0.0)));
let quantized =
i32x4_trunc_sat_f32x4(f32x4_add(truncated, v128_and(signed_one, ge_half)));
let quantized = v128_and(quantized, v128_and(keep, active));
unsafe { v128_store(levels.as_mut_ptr().add(y * width + x).cast(), quantized) };
}
if !SAVE_LEVELS && rate_bits != 0 {
let ratev = wasm_log2p1_f32(absq);
mag_acc = f32x4_add(mag_acc, v128_bitselect(ratev, zero, rate_mask));
let sv = unsafe { v128_load(scan_pos.as_ptr().add(y * width + x) as *const v128) };
scan_acc = u32x4_max(scan_acc, v128_and(sv, rate_mask));
}
}
}
let max_scan = u32x4_extract_lane::<0>(scan_acc)
.max(u32x4_extract_lane::<1>(scan_acc))
.max(u32x4_extract_lane::<2>(scan_acc))
.max(u32x4_extract_lane::<3>(scan_acc));
(hsum4(sse_acc), nzeros, hsum4(mag_acc), max_scan)
}
#[inline]
#[target_feature(enable = "simd128")]
pub(crate) fn wasm_log2p1_f32(x: v128) -> v128 {
let y = f32x4_add(x, f32x4_splat(1.0));
let bits = y;
let exp_u = u32x4_shr(bits, 23);
let exp_i = i32x4_sub(exp_u, i32x4_splat(127));
let e = f32x4_convert_i32x4(exp_i);
let mant_bits = v128_or(
v128_and(bits, i32x4_splat(0x007f_ffff)),
i32x4_splat(0x3f80_0000),
);
let m = mant_bits;
let t = f32x4_sub(m, f32x4_splat(1.0));
let c0 = f32x4_splat(1.4426934719085693359375);
let c1 = f32x4_splat(-0.721179187297821044921875);
let c2 = f32x4_splat(0.477900087833404541015625);
let c3 = f32x4_splat(-0.340080082416534423828125);
let c4 = f32x4_splat(0.21719777584075927734375);
let c5 = f32x4_splat(-9.749893844127655029296875e-2);
let c6 = f32x4_splat(2.096841670572757720947265625e-2);
let mut p = c6;
p = f32x4_add(c5, f32x4_mul(t, p));
p = f32x4_add(c4, f32x4_mul(t, p));
p = f32x4_add(c3, f32x4_mul(t, p));
p = f32x4_add(c2, f32x4_mul(t, p));
p = f32x4_add(c1, f32x4_mul(t, p));
p = f32x4_add(c0, f32x4_mul(t, p));
let log2_m = f32x4_mul(t, p);
f32x4_add(e, log2_m)
}
#[cfg(test)]
mod tests {
use super::sse_and_rate_wasm;
#[test]
fn test_sse_and_rate_wasm_vs_reference() {
crate::inflated_cost::assert_sse_and_rate_matches_reference(
sse_and_rate_wasm::<false>,
false,
);
crate::inflated_cost::assert_sse_and_rate_matches_reference(
sse_and_rate_wasm::<true>,
true,
);
}
}
#[cfg(test)]
mod lane_count_tests {
use super::*;
#[test]
fn interleaved_vectors_preserve_original_rate_distortion_and_coding_levels() {
for (original, optimized) in [
(
sse_and_rate_wasm_impl_reference::<false, false>
as crate::inflated_cost::SseAndQuantizeFn,
sse_and_rate_wasm_impl::<false, false> as crate::inflated_cost::SseAndQuantizeFn,
),
(
sse_and_rate_wasm_impl_reference::<false, true>
as crate::inflated_cost::SseAndQuantizeFn,
sse_and_rate_wasm_impl::<false, true> as crate::inflated_cost::SseAndQuantizeFn,
),
(
sse_and_rate_wasm_impl_reference::<true, false>
as crate::inflated_cost::SseAndQuantizeFn,
sse_and_rate_wasm_impl::<true, false> as crate::inflated_cost::SseAndQuantizeFn,
),
(
sse_and_rate_wasm_impl_reference::<true, true>
as crate::inflated_cost::SseAndQuantizeFn,
sse_and_rate_wasm_impl::<true, true> as crate::inflated_cost::SseAndQuantizeFn,
),
] {
crate::inflated_cost::assert_coefficient_kernel_preserves_original(original, optimized);
for width in [4, 12, 20] {
let n = width * 8;
let values = [0.0, 0.5, -0.5, 1.5, -1.5, 0.125];
let coeff: Vec<_> = (0..n).map(|i| values[i % values.len()]).collect();
let inverse = vec![1.0; n];
let scan: Vec<_> = (0..n as u32).collect();
let mut expected_levels = vec![-123; n + 5];
let mut actual_levels = expected_levels.clone();
let run = |kernel: crate::inflated_cost::SseAndQuantizeFn, levels: &mut [i32]| unsafe {
kernel(
&coeff,
&inverse,
1.0,
crate::group::DequantBias::DEFAULT[0],
width,
8,
width / 8 * 4,
1,
1,
crate::inflated_cost::rate_log2_lut(),
&[0.0, 0.42, 0.57, 0.91],
&scan,
levels,
)
};
let expected = run(original, &mut expected_levels);
let actual = run(optimized, &mut actual_levels);
assert_eq!(actual.0.to_bits(), expected.0.to_bits());
assert_eq!(actual.1, expected.1);
assert_eq!(actual.2.to_bits(), expected.2.to_bits());
assert_eq!(actual.3, expected.3);
assert_eq!(actual_levels, expected_levels);
}
}
}
}