use core::arch::wasm32::*;
#[inline]
#[target_feature(enable = "simd128")]
fn round_ties_away_x4(v: v128) -> v128 {
let half = f32x4_splat(0.5);
let positive = f32x4_ge(v, f32x4_splat(0.0));
let qp = f32x4_floor(f32x4_add(v, half));
let qn = f32x4_ceil(f32x4_sub(v, half));
v128_bitselect(qp, qn, positive)
}
#[inline]
#[target_feature(enable = "simd128")]
fn dequantized_level_x4(q: v128) -> v128 {
let absq = f32x4_abs(q);
let big = f32x4_sub(
q,
f32x4_div(f32x4_splat(crate::group::DEFAULT_QUANT_BIAS_3), q),
);
let sign = v128_and(q, i32x4_splat(i32::MIN));
let one = v128_or(sign, f32x4_splat(crate::group::DEFAULT_QUANT_BIAS_1));
let dq = v128_bitselect(big, one, f32x4_ge(absq, f32x4_splat(1.125)));
v128_and(dq, f32x4_gt(absq, f32x4_splat(0.0)))
}
#[target_feature(enable = "simd128")]
pub(crate) fn cfl_closed_loop_cost_wasm(
m: &[f32],
s: &[f32],
factor: f32,
thresholds: &[f32; 63],
) -> [f32; 2] {
let len = m.len().min(s.len());
let scalar_factor = factor;
let factor = f32x4_splat(factor);
let zero = f32x4_splat(0.0);
let one = f32x4_splat(1.0);
let mut distortion = 0.0f32;
let mut coeff_bits = 0.0f32;
let mut dist_acc = zero;
let mut bits_acc = zero;
let (m_chunks, m_tail) = m[..len].as_chunks::<4>();
let (s_chunks, s_tail) = s[..len].as_chunks::<4>();
for (chunk_index, (m, s)) in m_chunks.iter().zip(s_chunks).enumerate() {
let mv = unsafe { v128_load(m.as_ptr().cast()) };
let sv = unsafe { v128_load(s.as_ptr().cast()) };
let residual = f32x4_sub(sv, f32x4_mul(factor, mv));
let abs_residual = f32x4_abs(residual);
let phase = (chunk_index * 4) % 63;
let threshold = if phase + 4 <= 63 {
unsafe { v128_load(thresholds.as_ptr().add(phase).cast()) }
} else {
let wrapped = [
thresholds[phase],
thresholds[(phase + 1) % 63],
thresholds[(phase + 2) % 63],
thresholds[(phase + 3) % 63],
];
unsafe { v128_load(wrapped.as_ptr().cast()) }
};
let level = v128_bitselect(
round_ties_away_x4(residual),
zero,
f32x4_ge(abs_residual, threshold),
);
let reconstructed = dequantized_level_x4(level);
let quant_error = f32x4_abs(f32x4_sub(residual, reconstructed));
let dist = f32x4_add(quant_error, f32x4_mul(f32x4_splat(0.15), abs_residual));
let abs_level = f32x4_abs(level);
let bits = v128_bitselect(
f32x4_add(one, super::ac_strategy::wasm_log2p1_f32(abs_level)),
zero,
f32x4_gt(abs_level, zero),
);
dist_acc = f32x4_add(dist_acc, dist);
bits_acc = f32x4_add(bits_acc, bits);
}
distortion += super::frame::reduce_f32x4(dist_acc);
coeff_bits += super::frame::reduce_f32x4(bits_acc);
let tail_start = m_chunks.len() * 4;
for (offset, (&mv, &sv)) in m_tail.iter().zip(s_tail).enumerate() {
let residual = sv - scalar_factor * mv;
let level = if residual.abs() >= thresholds[(tail_start + offset) % 63] {
residual.round()
} else {
0.0
};
let quant_error = residual - crate::group::dequantized_level_f32(level);
distortion += quant_error.abs() + 0.15 * residual.abs();
if level != 0.0 {
coeff_bits += 1.0 + crate::adaptive_quant::dirty_log2p1f(level.abs());
}
}
[distortion, coeff_bits]
}
#[inline]
#[target_feature(enable = "simd128")]
fn load(values: &[f32; 4]) -> v128 {
unsafe { v128_load(values.as_ptr().cast()) }
}
#[inline]
#[target_feature(enable = "simd128")]
fn store(values: &mut [f32; 4], vector: v128) {
unsafe { v128_store(values.as_mut_ptr().cast(), vector) }
}
#[inline]
#[target_feature(enable = "simd128")]
fn reduce_transposed_4x4(a: v128, b: v128, c: v128, d: v128) -> [f32; 4] {
let ab_lo = i32x4_shuffle::<0, 4, 1, 5>(a, b);
let ab_hi = i32x4_shuffle::<2, 6, 3, 7>(a, b);
let cd_lo = i32x4_shuffle::<0, 4, 1, 5>(c, d);
let cd_hi = i32x4_shuffle::<2, 6, 3, 7>(c, d);
let lane_0 = i32x4_shuffle::<0, 1, 4, 5>(ab_lo, cd_lo);
let lane_1 = i32x4_shuffle::<2, 3, 6, 7>(ab_lo, cd_lo);
let lane_2 = i32x4_shuffle::<0, 1, 4, 5>(ab_hi, cd_hi);
let lane_3 = i32x4_shuffle::<2, 3, 6, 7>(ab_hi, cd_hi);
let sums = f32x4_add(f32x4_add(lane_0, lane_1), f32x4_add(lane_2, lane_3));
let mut out = [0.0f32; 4];
store(&mut out, sums);
out
}
#[target_feature(enable = "simd128")]
pub(crate) fn cfl_regression_wasm(
y: &[f32; 64],
x: &[f32; 64],
b: &[f32; 64],
qm_x: &[f32; 64],
qm_b: &[f32; 64],
) -> [f32; 4] {
let inv_color = f32x4_splat(1.0 / 84.0);
let mut ca_x = f32x4_splat(0.0);
let mut cb_x = f32x4_splat(0.0);
let mut ca_b = f32x4_splat(0.0);
let mut cb_b = f32x4_splat(0.0);
let y_chunks = y.as_chunks::<4>().0;
let x_chunks = x.as_chunks::<4>().0;
let b_chunks = b.as_chunks::<4>().0;
let qm_x_chunks = qm_x.as_chunks::<4>().0;
let qm_b_chunks = qm_b.as_chunks::<4>().0;
for ((((y, x), b), qm_x), qm_b) in y_chunks
.iter()
.zip(x_chunks)
.zip(b_chunks)
.zip(qm_x_chunks)
.zip(qm_b_chunks)
{
let yv = unsafe { v128_load(y.as_ptr().cast()) };
let xv = unsafe { v128_load(x.as_ptr().cast()) };
let bv = unsafe { v128_load(b.as_ptr().cast()) };
let qx = unsafe { v128_load(qm_x.as_ptr().cast()) };
let qb = unsafe { v128_load(qm_b.as_ptr().cast()) };
let mx = f32x4_mul(yv, qx);
let sx = f32x4_mul(xv, qx);
let ax = f32x4_mul(inv_color, mx);
ca_x = f32x4_add(ca_x, f32x4_mul(ax, ax));
cb_x = f32x4_sub(cb_x, f32x4_mul(ax, sx));
let mb = f32x4_mul(yv, qb);
let sb = f32x4_mul(bv, qb);
let ab = f32x4_mul(inv_color, mb);
let residual_b = f32x4_sub(mb, sb);
ca_b = f32x4_add(ca_b, f32x4_mul(ab, ab));
cb_b = f32x4_add(cb_b, f32x4_mul(ab, residual_b));
}
reduce_transposed_4x4(ca_x, cb_x, ca_b, cb_b)
}
#[target_feature(enable = "simd128")]
pub(crate) fn cfl_rdo_block_wasm(
m_x: &mut [f32; 63],
s_x: &mut [f32; 63],
m_b: &mut [f32; 63],
s_b: &mut [f32; 63],
block_y: &[f32; 64],
block_x: &[f32; 64],
block_b: &[f32; 64],
qm_x: &[f32; 64],
qm_b: &[f32; 64],
q_block: f32,
) {
let q_block_v = f32x4_splat(q_block);
for i in (0..60).step_by(4) {
let coeff = i + 1;
let y = unsafe { v128_load(block_y.as_ptr().add(coeff).cast()) };
let x = unsafe { v128_load(block_x.as_ptr().add(coeff).cast()) };
let b = unsafe { v128_load(block_b.as_ptr().add(coeff).cast()) };
let qx = unsafe { v128_load(qm_x.as_ptr().add(coeff).cast()) };
let qb = unsafe { v128_load(qm_b.as_ptr().add(coeff).cast()) };
let qx = f32x4_mul(qx, q_block_v);
let qb = f32x4_mul(qb, q_block_v);
unsafe {
v128_store(m_x.as_mut_ptr().add(i).cast(), f32x4_mul(y, qx));
v128_store(s_x.as_mut_ptr().add(i).cast(), f32x4_mul(x, qx));
v128_store(m_b.as_mut_ptr().add(i).cast(), f32x4_mul(y, qb));
v128_store(s_b.as_mut_ptr().add(i).cast(), f32x4_mul(b, qb));
}
}
let i = 59;
let coeff = i + 1;
let y = unsafe { v128_load(block_y.as_ptr().add(coeff).cast()) };
let x = unsafe { v128_load(block_x.as_ptr().add(coeff).cast()) };
let b = unsafe { v128_load(block_b.as_ptr().add(coeff).cast()) };
let qx = unsafe { v128_load(qm_x.as_ptr().add(coeff).cast()) };
let qb = unsafe { v128_load(qm_b.as_ptr().add(coeff).cast()) };
let qx = f32x4_mul(qx, q_block_v);
let qb = f32x4_mul(qb, q_block_v);
unsafe {
v128_store(m_x.as_mut_ptr().add(i).cast(), f32x4_mul(y, qx));
v128_store(s_x.as_mut_ptr().add(i).cast(), f32x4_mul(x, qx));
v128_store(m_b.as_mut_ptr().add(i).cast(), f32x4_mul(y, qb));
v128_store(s_b.as_mut_ptr().add(i).cast(), f32x4_mul(b, qb));
}
}
#[target_feature(enable = "simd128")]
pub(crate) fn cfl_rdo_stats_wasm(m: &[f32], s: &[f32]) -> [f32; 4] {
let len = m.len().min(s.len());
let (m_chunks, m_tail) = m[..len].as_chunks::<4>();
let (s_chunks, s_tail) = s[..len].as_chunks::<4>();
let mut dot_ms = f32x4_splat(0.0);
let mut dot_mm = f32x4_splat(0.0);
let mut dot_ss = f32x4_splat(0.0);
let mut sum_abs_s = f32x4_splat(0.0);
for (m, s) in m_chunks.iter().zip(s_chunks) {
let mv = unsafe { v128_load(m.as_ptr().cast()) };
let sv = unsafe { v128_load(s.as_ptr().cast()) };
dot_ms = f32x4_add(dot_ms, f32x4_mul(mv, sv));
dot_mm = f32x4_add(dot_mm, f32x4_mul(mv, mv));
dot_ss = f32x4_add(dot_ss, f32x4_mul(sv, sv));
sum_abs_s = f32x4_add(sum_abs_s, f32x4_abs(sv));
}
let mut stats = reduce_transposed_4x4(dot_ms, dot_mm, dot_ss, sum_abs_s);
for (&mv, &sv) in m_tail.iter().zip(s_tail) {
stats[0] += mv * sv;
stats[1] += mv * mv;
stats[2] += sv * sv;
stats[3] += sv.abs();
}
stats
}
#[target_feature(enable = "simd128")]
pub(crate) fn apply_cfl_wasm(x: &mut [f32], y: &[f32], b: &mut [f32], cmap_factor: [f32; 3]) {
assert_eq!(x.len(), y.len());
assert_eq!(x.len(), b.len());
let cx = f32x4_splat(-cmap_factor[0]);
let cb = f32x4_splat(-cmap_factor[2]);
let (x_chunks, x_tail) = x.as_chunks_mut::<4>();
let (y_chunks, y_tail) = y.as_chunks::<4>();
let (b_chunks, b_tail) = b.as_chunks_mut::<4>();
let (x_groups, x_chunks_tail) = x_chunks.as_chunks_mut::<4>();
let (y_groups, y_chunks_tail) = y_chunks.as_chunks::<4>();
let (b_groups, b_chunks_tail) = b_chunks.as_chunks_mut::<4>();
for ((x, y), b) in x_groups.iter_mut().zip(y_groups).zip(b_groups) {
let y0 = load(&y[0]);
let y1 = load(&y[1]);
let y2 = load(&y[2]);
let y3 = load(&y[3]);
let x0 = load(&x[0]);
let x1 = load(&x[1]);
let x2 = load(&x[2]);
let x3 = load(&x[3]);
let b0 = load(&b[0]);
let b1 = load(&b[1]);
let b2 = load(&b[2]);
let b3 = load(&b[3]);
let x0 = f32x4_add(x0, f32x4_mul(cx, y0));
let x1 = f32x4_add(x1, f32x4_mul(cx, y1));
let x2 = f32x4_add(x2, f32x4_mul(cx, y2));
let x3 = f32x4_add(x3, f32x4_mul(cx, y3));
let b0 = f32x4_add(b0, f32x4_mul(cb, y0));
let b1 = f32x4_add(b1, f32x4_mul(cb, y1));
let b2 = f32x4_add(b2, f32x4_mul(cb, y2));
let b3 = f32x4_add(b3, f32x4_mul(cb, y3));
store(&mut x[0], x0);
store(&mut x[1], x1);
store(&mut x[2], x2);
store(&mut x[3], x3);
store(&mut b[0], b0);
store(&mut b[1], b1);
store(&mut b[2], b2);
store(&mut b[3], b3);
}
for ((x, y), b) in x_chunks_tail
.iter_mut()
.zip(y_chunks_tail)
.zip(b_chunks_tail)
{
let yv = load(y);
store(x, f32x4_add(load(x), f32x4_mul(cx, yv)));
store(b, f32x4_add(load(b), f32x4_mul(cb, yv)));
}
for ((x, &y), b) in x_tail.iter_mut().zip(y_tail).zip(b_tail) {
*x -= cmap_factor[0] * y;
*b -= cmap_factor[2] * y;
}
}
#[cfg(test)]
mod tests {
#[test]
fn apply_cfl_wasm_matches_scalar() {
for size in [0, 1, 3, 4, 7, 16, 64, 1024, 4096] {
let y: Vec<f32> = (0..size)
.map(|i| ((i * 37 % 101) as f32 - 50.0) * 0.03125)
.collect();
let mut x: Vec<f32> = (0..size).map(|i| i as f32 * 0.015625 - 2.0).collect();
let mut b: Vec<f32> = (0..size).map(|i| 3.0 - i as f32 * 0.0078125).collect();
let mut want_x = x.clone();
let mut want_b = b.clone();
let factors: [f32; 3] = [0.3125, 0.0, -0.1875];
for i in 0..size {
want_x[i] += -factors[0] * y[i];
want_b[i] += -factors[2] * y[i];
}
super::apply_cfl_wasm(&mut x, &y, &mut b, factors);
assert_eq!(x, want_x, "X mismatch at size {size}");
assert_eq!(b, want_b, "B mismatch at size {size}");
}
}
}