simd/lib.rs
1// simd -- SSE2 / AVX2 registers behind one safe interface, so a kernel is written once and
2// runs on both instruction sets with bit-identical results (the assembly's BMV = 0 / 1
3// macros); ARM64 and the browser in place of Sse2: neon.rs, wasm.rs
4//
5// trait Simd a register type V and its operations, named after the SSE2 instructions;
6// Sse2 (128-bit registers) and Avx2 (256-bit registers used as two
7// independent 128-bit lanes: every operation works per 128-bit lane, so
8// lane k computes exactly what SSE2 computes for its data)
9// trait Kernel a computation generic over Simd (run::<S>)
10// dispatch(k) runs k with Avx2 when rt::cpu reports CPU_AVX2, else with Sse2
11// dispatch_sse2(k) always Sse2 (tests)
12// has_avx2()
13// copy_stream(dst, src) a copy whose stores bypass the caches (x86-64: movntdq), for big
14// results another thread reads later; it ends with the fence (sfence) the
15// stores need before anyone, the copying thread too, touches dst again
16// copy_stream_rows(dst, dst_stride, src, src_stride, width, rows) the same for a block of
17// rows with one fence at the end
18// stream_fence() sfence on its own (the copies above no longer need it)
19//
20// Pixel helpers work on "pixel lanes": the 4 bytes of a pixel zero-extended to 4 u16 lanes,
21// PX = 2 (SSE2) / 4 (AVX2) pixels per register, pixels 2k and 2k + 1 in 128-bit lane k.
22//
23// One of the four library crates that may contain `unsafe` (rt, simd, plat, gpu). Every
24// intrinsic is an unsafe call (core::arch requires the feature on the calling function, even
25// SSE2, the x86-64 baseline: sse!); Avx2 intrinsics are only reached through an Avx2 value,
26// which exists only after the CPU check (Avx2::new) -- the precondition of all of them.
27// Memory helpers check the slice lengths (panic, never an access outside the slice).
28// Kernels must not put closures or function pointers between dispatch and the intrinsics
29// (compiled outside the AVX2 context, every intrinsic would become a call); build_rs.py
30// refuses compute crates with out-of-line core_arch functions.
31#![no_std]
32
33// in a module's build (--cfg jflib_module, docs/CRATES.md) the module runtime goes by the same name
34#[cfg(jflib_module)]
35extern crate modrt as rt;
36
37#[cfg(target_arch = "x86_64")]
38mod native;
39#[cfg(target_arch = "x86_64")]
40pub use native::{Sse2, Avx2};
41mod scalar;
42pub use scalar::{Scalar, Scalar256};
43#[cfg(target_arch = "aarch64")] mod neon;
44#[cfg(target_arch = "aarch64")] pub use neon::{Neon128, Neon128 as Sse2};
45#[cfg(all(target_arch = "wasm32", target_feature = "simd128"))] mod wasm;
46#[cfg(all(target_arch = "wasm32", target_feature = "simd128"))] pub use wasm::{Wasm128, Wasm128 as Sse2};
47#[cfg(any(target_arch = "aarch64", all(target_arch = "wasm32", target_feature = "simd128")))]
48pub use scalar::Scalar256 as Avx2;
49#[cfg(not(any(target_arch = "x86_64", target_arch = "aarch64", all(target_arch = "wasm32", target_feature = "simd128"))))]
50pub use scalar::{Scalar as Sse2, Scalar256 as Avx2};
51
52pub use rt::cpu::CPU_AVX2;
53
54/// AVX2 kernels may be used (CPU and OS support, not disabled by `--simd sse2`)
55#[inline]
56pub fn has_avx2() -> bool {
57 rt::cpu::features() & CPU_AVX2 != 0
58}
59
60/// copy src into dst (the shorter length) with non-temporal stores where the target has them
61/// (x86-64: the 16-byte aligned part of dst): dst is neither read first nor pulled into the
62/// caches, which keep what the thread works on. Ends with the fence the stores need (sfence)
63/// before anything touches dst again; copy_stream_rows fences a block of rows once.
64pub fn copy_stream(dst: &mut [u8], src: &[u8]) {
65 // SAFETY: the fence follows at once.
66 if unsafe { copy_nt(dst, src) } {
67 stream_fence();
68 }
69}
70
71/// copy_stream for `rows` rows of `width` bytes: row y of src starts at y * src_stride, of dst
72/// at y * dst_stride; one fence at the end. Panics when a row lies outside its slice.
73pub fn copy_stream_rows(dst: &mut [u8], dst_stride: usize, src: &[u8], src_stride: usize, width: usize, rows: usize) {
74 // checked before the first store: a panic between the stores and the fence would leave
75 // them unfenced while the unwinding goes on
76 let fits = |len: usize, stride: usize| rows == 0 || (rows - 1).checked_mul(stride).and_then(|o| o.checked_add(width)).is_some_and(|end| end <= len);
77 assert!(fits(dst.len(), dst_stride) && fits(src.len(), src_stride), "copy_stream_rows: a row lies outside its slice");
78 let mut streamed = false;
79 for y in 0..rows {
80 let (d, s) = (y * dst_stride, y * src_stride);
81 // SAFETY: the fence follows after the last row, before this function returns.
82 streamed |= unsafe { copy_nt(&mut dst[d..d + width], &src[s..s + width]) };
83 }
84 if streamed {
85 stream_fence();
86 }
87}
88
89/// copy_stream without its fence -> whether non-temporal stores were used
90///
91/// # Safety
92/// When it returns true, stream_fence() must run on this thread before anything (this thread
93/// included) reads or writes dst again: Rust's memory model does not cover the stores before it.
94#[cfg(target_arch = "x86_64")]
95unsafe fn copy_nt(dst: &mut [u8], src: &[u8]) -> bool {
96 use core::arch::x86_64::{__m128i, _mm_loadu_si128, _mm_stream_si128};
97 let n = dst.len().min(src.len());
98 let (dst, src) = (&mut dst[..n], &src[..n]);
99 let head = (dst.as_ptr() as usize).wrapping_neg() & 15;
100 if n < head + 64 {
101 dst.copy_from_slice(src);
102 return false;
103 }
104 dst[..head].copy_from_slice(&src[..head]);
105 let body = (n - head) & !63;
106 let (d, s) = (&mut dst[head..head + body], &src[head..head + body]);
107 let mut i = 0;
108 while i < body {
109 // SAFETY: i + 64 <= body: 64 readable bytes of s and 64 writable bytes of d at i; d's
110 // start is 16-byte aligned (head), so every store address is (movntdq's alignment).
111 unsafe {
112 let (sp, dp) = (s.as_ptr().add(i) as *const __m128i, d.as_mut_ptr().add(i) as *mut __m128i);
113 let (a, b, c, e) = (_mm_loadu_si128(sp), _mm_loadu_si128(sp.add(1)), _mm_loadu_si128(sp.add(2)), _mm_loadu_si128(sp.add(3)));
114 _mm_stream_si128(dp, a);
115 _mm_stream_si128(dp.add(1), b);
116 _mm_stream_si128(dp.add(2), c);
117 _mm_stream_si128(dp.add(3), e);
118 }
119 i += 64;
120 }
121 dst[head + body..].copy_from_slice(&src[head + body..]);
122 true
123}
124
125/// copy_stream without streaming stores (other targets). ARM64 needs nothing of its own: stnp
126/// is only a hint and its loose ordering would want a barrier, while a copy like this one
127/// writes whole lines in a row, which ARM cores notice and then stop allocating in their caches.
128///
129/// # Safety
130/// None here (it is unsafe for the x86-64 version's sake).
131#[cfg(not(target_arch = "x86_64"))]
132unsafe fn copy_nt(dst: &mut [u8], src: &[u8]) -> bool {
133 let n = dst.len().min(src.len());
134 dst[..n].copy_from_slice(&src[..n]);
135 false
136}
137
138/// the streaming stores of this thread become visible to others (sfence); copy_stream and
139/// copy_stream_rows run it themselves. Nothing to do on other targets: without such stores the
140/// release of whatever hands dst on is enough
141#[inline]
142pub fn stream_fence() {
143 // SAFETY: sfence is part of the x86-64 baseline (SSE); it only orders stores.
144 #[cfg(target_arch = "x86_64")]
145 unsafe {
146 core::arch::x86_64::_mm_sfence()
147 };
148}
149
150/// a computation written once for every instruction set
151pub trait Kernel {
152 type Out;
153 fn run<S: Simd>(self, s: S) -> Self::Out;
154}
155
156/// run k with the best instruction set of this CPU
157#[inline]
158#[cfg(target_arch = "x86_64")]
159pub fn dispatch<K: Kernel>(k: K) -> K::Out {
160 match Avx2::new() {
161 // SAFETY: Avx2::new succeeded, so the CPU and the OS support AVX2.
162 Some(a) => unsafe { run_avx2(k, a) },
163 None => k.run(Sse2),
164 }
165}
166
167/// run k with SSE2 (the baseline; tests compare it with dispatch)
168#[inline]
169pub fn dispatch_sse2<K: Kernel>(k: K) -> K::Out {
170 k.run(Sse2)
171}
172
173/// Run a kernel with NEON on ARM64, with SIMD128 when enabled by the WASM bundle, otherwise
174/// scalar registers.
175#[cfg(not(target_arch = "x86_64"))]
176#[inline]
177pub fn dispatch<K: Kernel>(k: K) -> K::Out { k.run(Sse2) }
178
179// the kernel compiled with AVX2 enabled (its #[inline(always)] code is inlined here)
180#[cfg(target_arch = "x86_64")]
181#[target_feature(enable = "avx2")]
182fn run_avx2<K: Kernel>(k: K, a: Avx2) -> K::Out {
183 k.run(a)
184}
185
186/// operations of one register type; integer lanes are named by their width (8 / 16 / 32),
187/// signed operations end in s, unsigned in u
188pub trait Simd: Copy {
189 type V: Copy;
190 /// bytes per register
191 const BYTES: usize;
192 /// pixels per register in pixel lanes
193 const PX: usize;
194
195 fn zero(self) -> Self::V;
196 fn splat8(self, v: u8) -> Self::V;
197 fn splat16(self, v: u16) -> Self::V;
198 fn splat32(self, v: u32) -> Self::V;
199 fn splat64(self, v: u64) -> Self::V;
200 fn splatf(self, v: f32) -> Self::V;
201
202 // ---- memory
203 /// BYTES bytes
204 fn load(self, b: &[u8]) -> Self::V;
205 fn store(self, b: &mut [u8], v: Self::V);
206 /// 4 * PX bytes (PX pixels) zero-extended to pixel lanes
207 fn load_px(self, b: &[u8]) -> Self::V;
208 /// pixel lanes saturated to bytes (packuswb), 4 * PX bytes
209 fn store_px(self, b: &mut [u8], v: Self::V);
210 /// PX bytes, byte i to the 4 u16 lanes of pixel i
211 fn bcast_px(self, b: &[u8]) -> Self::V;
212 /// 4 * PX bytes as they are ("raw" pixels: SSE2 the low 8 bytes, AVX2 the low 16)
213 fn load_raw(self, b: &[u8]) -> Self::V;
214 fn store_raw(self, b: &mut [u8], v: Self::V);
215 /// raw pixels to pixel lanes
216 fn widen_px(self, v: Self::V) -> Self::V;
217 /// 4 bytes into the low dword of every 128-bit lane (BYTES / 4 bytes), the rest 0
218 fn load_lane32(self, b: &[u8]) -> Self::V;
219 /// BYTES / 4 i32 values
220 fn load_i32(self, b: &[i32]) -> Self::V;
221 fn store_i32(self, b: &mut [i32], v: Self::V);
222 /// movd: v in dword 0, the rest 0 / dword 0 of v
223 fn from_i32(self, v: i32) -> Self::V;
224 fn to_i32(self, v: Self::V) -> i32;
225
226 // ---- 8-bit lanes
227 fn add8(self, a: Self::V, b: Self::V) -> Self::V;
228 fn sub8(self, a: Self::V, b: Self::V) -> Self::V;
229 fn addus8(self, a: Self::V, b: Self::V) -> Self::V;
230 fn subus8(self, a: Self::V, b: Self::V) -> Self::V;
231 fn maxu8(self, a: Self::V, b: Self::V) -> Self::V;
232 fn minu8(self, a: Self::V, b: Self::V) -> Self::V;
233 fn cmpeq8(self, a: Self::V, b: Self::V) -> Self::V;
234 fn avgu8(self, a: Self::V, b: Self::V) -> Self::V;
235 /// bit i = top bit of byte i (BYTES bits)
236 fn movemask8(self, a: Self::V) -> u32;
237 fn unpacklo8(self, a: Self::V, b: Self::V) -> Self::V;
238 fn unpackhi8(self, a: Self::V, b: Self::V) -> Self::V;
239 fn packus16(self, a: Self::V, b: Self::V) -> Self::V;
240
241 // ---- 16-bit lanes
242 fn add16(self, a: Self::V, b: Self::V) -> Self::V;
243 fn sub16(self, a: Self::V, b: Self::V) -> Self::V;
244 fn subus16(self, a: Self::V, b: Self::V) -> Self::V;
245 fn mullo16(self, a: Self::V, b: Self::V) -> Self::V;
246 fn mulhiu16(self, a: Self::V, b: Self::V) -> Self::V;
247 fn mins16(self, a: Self::V, b: Self::V) -> Self::V;
248 fn maxs16(self, a: Self::V, b: Self::V) -> Self::V;
249 fn cmpeq16(self, a: Self::V, b: Self::V) -> Self::V;
250 fn cmpgts16(self, a: Self::V, b: Self::V) -> Self::V;
251 fn srl16<const N: i32>(self, a: Self::V) -> Self::V;
252 fn sll16<const N: i32>(self, a: Self::V) -> Self::V;
253 /// pshuflw + pshufhw with the same immediate (per 64-bit half)
254 fn shuf16<const IMM: i32>(self, a: Self::V) -> Self::V;
255 fn unpacklo16(self, a: Self::V, b: Self::V) -> Self::V;
256 fn unpackhi16(self, a: Self::V, b: Self::V) -> Self::V;
257 fn packs32(self, a: Self::V, b: Self::V) -> Self::V;
258 /// pmaddwd: i32 lanes = a0 * b0 + a1 * b1 of the signed 16-bit pairs
259 fn madd16(self, a: Self::V, b: Self::V) -> Self::V;
260
261 // ---- 32-bit lanes
262 fn add32(self, a: Self::V, b: Self::V) -> Self::V;
263 fn sub32(self, a: Self::V, b: Self::V) -> Self::V;
264 fn cmpeq32(self, a: Self::V, b: Self::V) -> Self::V;
265 fn cmpgts32(self, a: Self::V, b: Self::V) -> Self::V;
266 /// bit i = top bit of dword i (BYTES / 4 bits)
267 fn movemask32(self, a: Self::V) -> u32;
268 /// pshufd (per 128-bit lane)
269 fn shuf32<const IMM: i32>(self, a: Self::V) -> Self::V;
270 /// shufps on integer registers: dwords 0, 1 from a, 2, 3 from b (per 128-bit lane)
271 fn shufps<const IMM: i32>(self, a: Self::V, b: Self::V) -> Self::V;
272 fn srl32<const N: i32>(self, a: Self::V) -> Self::V;
273 fn sll32<const N: i32>(self, a: Self::V) -> Self::V;
274 fn sra32<const N: i32>(self, a: Self::V) -> Self::V;
275 /// pslldq / psrldq: byte shifts per 128-bit lane
276 fn bsll<const N: i32>(self, a: Self::V) -> Self::V;
277 fn bsrl<const N: i32>(self, a: Self::V) -> Self::V;
278
279 // ---- bits
280 fn and(self, a: Self::V, b: Self::V) -> Self::V;
281 /// !a & b (pandn)
282 fn andnot(self, a: Self::V, b: Self::V) -> Self::V;
283 fn or(self, a: Self::V, b: Self::V) -> Self::V;
284 fn xor(self, a: Self::V, b: Self::V) -> Self::V;
285
286 // ---- f32 lanes
287 fn cvt_i32_f32(self, a: Self::V) -> Self::V;
288 /// truncating (0x80000000 for NaN / out of range)
289 fn cvtt_f32_i32(self, a: Self::V) -> Self::V;
290 /// rounding to nearest even (cvtps2dq)
291 fn cvtr_f32_i32(self, a: Self::V) -> Self::V;
292 fn addf(self, a: Self::V, b: Self::V) -> Self::V;
293 fn subf(self, a: Self::V, b: Self::V) -> Self::V;
294 fn mulf(self, a: Self::V, b: Self::V) -> Self::V;
295 fn divf(self, a: Self::V, b: Self::V) -> Self::V;
296
297 // ---- f64 lanes (BYTES / 8 of them); bit operations: and / andnot / or / xor
298 fn splatd(self, v: f64) -> Self::V;
299 fn addd(self, a: Self::V, b: Self::V) -> Self::V;
300 fn subd(self, a: Self::V, b: Self::V) -> Self::V;
301 fn muld(self, a: Self::V, b: Self::V) -> Self::V;
302 fn divd(self, a: Self::V, b: Self::V) -> Self::V;
303 fn sqrtd(self, a: Self::V) -> Self::V;
304 /// maxpd / minpd: b when unordered
305 fn maxd(self, a: Self::V, b: Self::V) -> Self::V;
306 fn mind(self, a: Self::V, b: Self::V) -> Self::V;
307 /// cmpltpd: all ones where a < b (false when unordered)
308 fn cmpltd(self, a: Self::V, b: Self::V) -> Self::V;
309 /// cvtdq2pd: the first BYTES / 8 i32 lanes to f64
310 fn cvt_i32_f64(self, a: Self::V) -> Self::V;
311 /// cvttpd2dq: f64 lanes to the first BYTES / 8 i32 lanes (truncating, 0x80000000 when
312 /// out of range), the rest 0
313 fn cvtt_f64_i32(self, a: Self::V) -> Self::V;
314 /// the first BYTES / 8 i32 lanes
315 fn store_i32_half(self, b: &mut [i32], v: Self::V);
316
317 // R6-jpeg, R6-xf: the JPEG kernels (img_jpeg dsp)
318 /// psraw: arithmetic right shift of the i16 lanes
319 fn sra16<const N: i32>(self, a: Self::V) -> Self::V;
320 /// pmulhw: high 16 bits of the signed 16-bit products
321 fn mulhis16(self, a: Self::V, b: Self::V) -> Self::V;
322 /// packsswb: i16 lanes of a, then b, saturated to i8
323 fn packs16(self, a: Self::V, b: Self::V) -> Self::V;
324 /// BYTES / 2 i16 values
325 fn load_i16(self, b: &[i16]) -> Self::V;
326 fn store_i16(self, b: &mut [i16], v: Self::V);
327 /// 2 * BYTES bytes: per 128-bit lane k, lane k of lo then lane k of hi (SSE2: lo, hi;
328 /// AVX2: vperm2i128) -- 2 x 4 pixels per lane in pixel order
329 fn store_lanes2(self, b: &mut [u8], lo: Self::V, hi: Self::V);
330 /// the inverse of store_lanes2: 2 * BYTES bytes -> (lo, hi)
331 fn load_lanes2(self, b: &[u8]) -> (Self::V, Self::V);
332 /// 8 i16 per 128-bit lane from separate sources: lane 0 from a, lane 1 from b (SSE2: a)
333 fn load_i16_lanes(self, a: &[i16], b: &[i16]) -> Self::V;
334 /// the low 8 bytes of every 128-bit lane to separate places: lane 0 to a, lane 1 to b
335 /// (SSE2: a)
336 fn store_raw_lanes(self, a: &mut [u8], b: &mut [u8], v: Self::V);
337 /// (main agent) psadbw: per 8-byte group the sum of |a - b| (u8) in its u64 lane
338 fn sadu8(self, a: Self::V, b: Self::V) -> Self::V;
339 /// (main agent) u64 lanes
340 fn add64(self, a: Self::V, b: Self::V) -> Self::V;
341
342 // R6-icc: the ICC transform kernels (img_core icc_xform)
343 /// maxps / minps: b when unordered
344 fn maxf(self, a: Self::V, b: Self::V) -> Self::V;
345 fn minf(self, a: Self::V, b: Self::V) -> Self::V;
346 /// sqrtps
347 fn sqrtf(self, a: Self::V) -> Self::V;
348
349 // r9-av1: the CDEF kernel (img_av1 cdef)
350 /// BYTES / 2 u16 values
351 fn load_u16(self, b: &[u16]) -> Self::V;
352 fn store_u16(self, b: &mut [u16], v: Self::V);
353 /// movq: 4 u16 values into the low 64 bits (the rest 0) / the low 64 bits to 4 u16 values
354 fn load_u16_lo(self, b: &[u16]) -> Self::V;
355 fn store_u16_lo(self, b: &mut [u16], v: Self::V);
356 /// psrlw with a register count: logical right shift of the 16-bit lanes by n (n > 15: 0)
357 fn srl16x(self, a: Self::V, n: u32) -> Self::V;
358
359 // R9-jxl: the JPEG XL render kernels (img_jxl)
360 /// BYTES / 4 f32 values
361 fn load_f32(self, b: &[f32]) -> Self::V;
362 fn store_f32(self, b: &mut [f32], v: Self::V);
363
364 // R9-webp: the VP8 loop filter (img_webp)
365 /// paddsb: signed saturating add of the i8 lanes
366 fn adds8(self, a: Self::V, b: Self::V) -> Self::V;
367 /// psubsb: signed saturating subtract of the i8 lanes
368 fn subs8(self, a: Self::V, b: Self::V) -> Self::V;
369
370 // r9-dds: the BC encoders' pixel-set helpers (img_dds enc)
371 /// cmpltps: all ones where a < b (false when unordered)
372 fn cmpltf(self, a: Self::V, b: Self::V) -> Self::V;
373
374 // jxl-speed: the JPEG XL writer's transforms (img_jxl dctk: 4 x 4 transposes per lane)
375 /// 4 f32 per 128-bit lane from separate sources: lane 0 from a, lane 1 from b (SSE2: a)
376 fn load_f32_lanes(self, a: &[f32], b: &[f32]) -> Self::V;
377 /// the 4 f32 of every 128-bit lane to separate places: lane 0 to a, lane 1 to b (SSE2: a)
378 fn store_f32_lanes(self, a: &mut [f32], b: &mut [f32], v: Self::V);
379
380 // perf-blend: the blend-mode kernels (gfx blendmode). core::arch writes the pack
381 // intrinsics as generic smax / smin + shuffle; with AVX2 (SSE4.1's pmaxsd / pminsd
382 // available) LLVM keeps the clamps in front of the pack: 4 (packs32) / 2 (packus16) more
383 // instructions per pack. On AVX2 these two are exactly one pack instruction (inline
384 // assembly); SSE2 uses the intrinsics (packs32 is one instruction there, packus16 keeps
385 // pmaxsw / pminsw: legacy-encoded inline assembly would hurt AVX2 code calling Sse2).
386 /// packs32 (packssdw), on AVX2 without the clamps
387 fn packssdw(self, a: Self::V, b: Self::V) -> Self::V;
388 /// pixel lanes to raw pixels: packuswb (+ vpermq 0x08 on AVX2) as in store_px, on AVX2
389 /// without the clamps; the inverse of widen_px
390 fn packuswb_px(self, v: Self::V) -> Self::V;
391 /// ptest on the raw-pixel bytes (SSE2: the low 8, AVX2: the low 16): 1 when a & b == 0
392 /// (ZF), else 2 when !a & b == 0 (CF), else 0. AVX2: vptest + jz + jnc, the CF case
393 /// falls through (inline assembly: core::arch writes ptest as or-reductions, and LLVM
394 /// then often builds shuffle / reduce sequences or loses the CF test); branch on the
395 /// result, the branches thread through
396 fn ptest_raw(self, a: Self::V, b: Self::V) -> u32;
397 /// ptest ZF on the raw-pixel bytes: a & b == 0 (AVX2: vptest + jz)
398 fn ptestz_raw(self, a: Self::V, b: Self::V) -> bool;
399}