Skip to main content

simd/
lib.rs

1// simd -- SSE2 / AVX2 registers behind one safe interface, so a kernel is written once and
2// runs on both instruction sets with bit-identical results (the assembly's BMV = 0 / 1
3// macros); ARM64 and the browser in place of Sse2: neon.rs, wasm.rs
4//
5//   trait Simd       a register type V and its operations, named after the SSE2 instructions;
6//                    Sse2 (128-bit registers) and Avx2 (256-bit registers used as two
7//                    independent 128-bit lanes: every operation works per 128-bit lane, so
8//                    lane k computes exactly what SSE2 computes for its data)
9//   trait Kernel     a computation generic over Simd (run::<S>)
10//   dispatch(k)      runs k with Avx2 when rt::cpu reports CPU_AVX2, else with Sse2
11//   dispatch_sse2(k) always Sse2 (tests)
12//   has_avx2()
13//   copy_stream(dst, src)   a copy whose stores bypass the caches (x86-64: movntdq), for big
14//                    results another thread reads later; it ends with the fence (sfence) the
15//                    stores need before anyone, the copying thread too, touches dst again
16//   copy_stream_rows(dst, dst_stride, src, src_stride, width, rows)   the same for a block of
17//                    rows with one fence at the end
18//   stream_fence()   sfence on its own (the copies above no longer need it)
19//
20// Pixel helpers work on "pixel lanes": the 4 bytes of a pixel zero-extended to 4 u16 lanes,
21// PX = 2 (SSE2) / 4 (AVX2) pixels per register, pixels 2k and 2k + 1 in 128-bit lane k.
22//
23// One of the four library crates that may contain `unsafe` (rt, simd, plat, gpu).  Every
24// intrinsic is an unsafe call (core::arch requires the feature on the calling function, even
25// SSE2, the x86-64 baseline: sse!); Avx2 intrinsics are only reached through an Avx2 value,
26// which exists only after the CPU check (Avx2::new) -- the precondition of all of them.
27// Memory helpers check the slice lengths (panic, never an access outside the slice).
28// Kernels must not put closures or function pointers between dispatch and the intrinsics
29// (compiled outside the AVX2 context, every intrinsic would become a call); build_rs.py
30// refuses compute crates with out-of-line core_arch functions.
31#![no_std]
32
33// in a module's build (--cfg jflib_module, docs/CRATES.md) the module runtime goes by the same name
34#[cfg(jflib_module)]
35extern crate modrt as rt;
36
37#[cfg(target_arch = "x86_64")]
38mod native;
39#[cfg(target_arch = "x86_64")]
40pub use native::{Sse2, Avx2};
41mod scalar;
42pub use scalar::{Scalar, Scalar256};
43#[cfg(target_arch = "aarch64")] mod neon;
44#[cfg(target_arch = "aarch64")] pub use neon::{Neon128, Neon128 as Sse2};
45#[cfg(all(target_arch = "wasm32", target_feature = "simd128"))] mod wasm;
46#[cfg(all(target_arch = "wasm32", target_feature = "simd128"))] pub use wasm::{Wasm128, Wasm128 as Sse2};
47#[cfg(any(target_arch = "aarch64", all(target_arch = "wasm32", target_feature = "simd128")))]
48pub use scalar::Scalar256 as Avx2;
49#[cfg(not(any(target_arch = "x86_64", target_arch = "aarch64", all(target_arch = "wasm32", target_feature = "simd128"))))]
50pub use scalar::{Scalar as Sse2, Scalar256 as Avx2};
51
52pub use rt::cpu::CPU_AVX2;
53
54/// AVX2 kernels may be used (CPU and OS support, not disabled by `--simd sse2`)
55#[inline]
56pub fn has_avx2() -> bool {
57    rt::cpu::features() & CPU_AVX2 != 0
58}
59
60/// copy src into dst (the shorter length) with non-temporal stores where the target has them
61/// (x86-64: the 16-byte aligned part of dst): dst is neither read first nor pulled into the
62/// caches, which keep what the thread works on.  Ends with the fence the stores need (sfence)
63/// before anything touches dst again; copy_stream_rows fences a block of rows once.
64pub fn copy_stream(dst: &mut [u8], src: &[u8]) {
65    // SAFETY: the fence follows at once.
66    if unsafe { copy_nt(dst, src) } {
67        stream_fence();
68    }
69}
70
71/// copy_stream for `rows` rows of `width` bytes: row y of src starts at y * src_stride, of dst
72/// at y * dst_stride; one fence at the end.  Panics when a row lies outside its slice.
73pub fn copy_stream_rows(dst: &mut [u8], dst_stride: usize, src: &[u8], src_stride: usize, width: usize, rows: usize) {
74    // checked before the first store: a panic between the stores and the fence would leave
75    // them unfenced while the unwinding goes on
76    let fits = |len: usize, stride: usize| rows == 0 || (rows - 1).checked_mul(stride).and_then(|o| o.checked_add(width)).is_some_and(|end| end <= len);
77    assert!(fits(dst.len(), dst_stride) && fits(src.len(), src_stride), "copy_stream_rows: a row lies outside its slice");
78    let mut streamed = false;
79    for y in 0..rows {
80        let (d, s) = (y * dst_stride, y * src_stride);
81        // SAFETY: the fence follows after the last row, before this function returns.
82        streamed |= unsafe { copy_nt(&mut dst[d..d + width], &src[s..s + width]) };
83    }
84    if streamed {
85        stream_fence();
86    }
87}
88
89/// copy_stream without its fence -> whether non-temporal stores were used
90///
91/// # Safety
92/// When it returns true, stream_fence() must run on this thread before anything (this thread
93/// included) reads or writes dst again: Rust's memory model does not cover the stores before it.
94#[cfg(target_arch = "x86_64")]
95unsafe fn copy_nt(dst: &mut [u8], src: &[u8]) -> bool {
96    use core::arch::x86_64::{__m128i, _mm_loadu_si128, _mm_stream_si128};
97    let n = dst.len().min(src.len());
98    let (dst, src) = (&mut dst[..n], &src[..n]);
99    let head = (dst.as_ptr() as usize).wrapping_neg() & 15;
100    if n < head + 64 {
101        dst.copy_from_slice(src);
102        return false;
103    }
104    dst[..head].copy_from_slice(&src[..head]);
105    let body = (n - head) & !63;
106    let (d, s) = (&mut dst[head..head + body], &src[head..head + body]);
107    let mut i = 0;
108    while i < body {
109        // SAFETY: i + 64 <= body: 64 readable bytes of s and 64 writable bytes of d at i; d's
110        // start is 16-byte aligned (head), so every store address is (movntdq's alignment).
111        unsafe {
112            let (sp, dp) = (s.as_ptr().add(i) as *const __m128i, d.as_mut_ptr().add(i) as *mut __m128i);
113            let (a, b, c, e) = (_mm_loadu_si128(sp), _mm_loadu_si128(sp.add(1)), _mm_loadu_si128(sp.add(2)), _mm_loadu_si128(sp.add(3)));
114            _mm_stream_si128(dp, a);
115            _mm_stream_si128(dp.add(1), b);
116            _mm_stream_si128(dp.add(2), c);
117            _mm_stream_si128(dp.add(3), e);
118        }
119        i += 64;
120    }
121    dst[head + body..].copy_from_slice(&src[head + body..]);
122    true
123}
124
125/// copy_stream without streaming stores (other targets).  ARM64 needs nothing of its own: stnp
126/// is only a hint and its loose ordering would want a barrier, while a copy like this one
127/// writes whole lines in a row, which ARM cores notice and then stop allocating in their caches.
128///
129/// # Safety
130/// None here (it is unsafe for the x86-64 version's sake).
131#[cfg(not(target_arch = "x86_64"))]
132unsafe fn copy_nt(dst: &mut [u8], src: &[u8]) -> bool {
133    let n = dst.len().min(src.len());
134    dst[..n].copy_from_slice(&src[..n]);
135    false
136}
137
138/// the streaming stores of this thread become visible to others (sfence); copy_stream and
139/// copy_stream_rows run it themselves.  Nothing to do on other targets: without such stores the
140/// release of whatever hands dst on is enough
141#[inline]
142pub fn stream_fence() {
143    // SAFETY: sfence is part of the x86-64 baseline (SSE); it only orders stores.
144    #[cfg(target_arch = "x86_64")]
145    unsafe {
146        core::arch::x86_64::_mm_sfence()
147    };
148}
149
150/// a computation written once for every instruction set
151pub trait Kernel {
152    type Out;
153    fn run<S: Simd>(self, s: S) -> Self::Out;
154}
155
156/// run k with the best instruction set of this CPU
157#[inline]
158#[cfg(target_arch = "x86_64")]
159pub fn dispatch<K: Kernel>(k: K) -> K::Out {
160    match Avx2::new() {
161        // SAFETY: Avx2::new succeeded, so the CPU and the OS support AVX2.
162        Some(a) => unsafe { run_avx2(k, a) },
163        None => k.run(Sse2),
164    }
165}
166
167/// run k with SSE2 (the baseline; tests compare it with dispatch)
168#[inline]
169pub fn dispatch_sse2<K: Kernel>(k: K) -> K::Out {
170    k.run(Sse2)
171}
172
173/// Run a kernel with NEON on ARM64, with SIMD128 when enabled by the WASM bundle, otherwise
174/// scalar registers.
175#[cfg(not(target_arch = "x86_64"))]
176#[inline]
177pub fn dispatch<K: Kernel>(k: K) -> K::Out { k.run(Sse2) }
178
179// the kernel compiled with AVX2 enabled (its #[inline(always)] code is inlined here)
180#[cfg(target_arch = "x86_64")]
181#[target_feature(enable = "avx2")]
182fn run_avx2<K: Kernel>(k: K, a: Avx2) -> K::Out {
183    k.run(a)
184}
185
186/// operations of one register type; integer lanes are named by their width (8 / 16 / 32),
187/// signed operations end in s, unsigned in u
188pub trait Simd: Copy {
189    type V: Copy;
190    /// bytes per register
191    const BYTES: usize;
192    /// pixels per register in pixel lanes
193    const PX: usize;
194
195    fn zero(self) -> Self::V;
196    fn splat8(self, v: u8) -> Self::V;
197    fn splat16(self, v: u16) -> Self::V;
198    fn splat32(self, v: u32) -> Self::V;
199    fn splat64(self, v: u64) -> Self::V;
200    fn splatf(self, v: f32) -> Self::V;
201
202    // ---- memory
203    /// BYTES bytes
204    fn load(self, b: &[u8]) -> Self::V;
205    fn store(self, b: &mut [u8], v: Self::V);
206    /// 4 * PX bytes (PX pixels) zero-extended to pixel lanes
207    fn load_px(self, b: &[u8]) -> Self::V;
208    /// pixel lanes saturated to bytes (packuswb), 4 * PX bytes
209    fn store_px(self, b: &mut [u8], v: Self::V);
210    /// PX bytes, byte i to the 4 u16 lanes of pixel i
211    fn bcast_px(self, b: &[u8]) -> Self::V;
212    /// 4 * PX bytes as they are ("raw" pixels: SSE2 the low 8 bytes, AVX2 the low 16)
213    fn load_raw(self, b: &[u8]) -> Self::V;
214    fn store_raw(self, b: &mut [u8], v: Self::V);
215    /// raw pixels to pixel lanes
216    fn widen_px(self, v: Self::V) -> Self::V;
217    /// 4 bytes into the low dword of every 128-bit lane (BYTES / 4 bytes), the rest 0
218    fn load_lane32(self, b: &[u8]) -> Self::V;
219    /// BYTES / 4 i32 values
220    fn load_i32(self, b: &[i32]) -> Self::V;
221    fn store_i32(self, b: &mut [i32], v: Self::V);
222    /// movd: v in dword 0, the rest 0 / dword 0 of v
223    fn from_i32(self, v: i32) -> Self::V;
224    fn to_i32(self, v: Self::V) -> i32;
225
226    // ---- 8-bit lanes
227    fn add8(self, a: Self::V, b: Self::V) -> Self::V;
228    fn sub8(self, a: Self::V, b: Self::V) -> Self::V;
229    fn addus8(self, a: Self::V, b: Self::V) -> Self::V;
230    fn subus8(self, a: Self::V, b: Self::V) -> Self::V;
231    fn maxu8(self, a: Self::V, b: Self::V) -> Self::V;
232    fn minu8(self, a: Self::V, b: Self::V) -> Self::V;
233    fn cmpeq8(self, a: Self::V, b: Self::V) -> Self::V;
234    fn avgu8(self, a: Self::V, b: Self::V) -> Self::V;
235    /// bit i = top bit of byte i (BYTES bits)
236    fn movemask8(self, a: Self::V) -> u32;
237    fn unpacklo8(self, a: Self::V, b: Self::V) -> Self::V;
238    fn unpackhi8(self, a: Self::V, b: Self::V) -> Self::V;
239    fn packus16(self, a: Self::V, b: Self::V) -> Self::V;
240
241    // ---- 16-bit lanes
242    fn add16(self, a: Self::V, b: Self::V) -> Self::V;
243    fn sub16(self, a: Self::V, b: Self::V) -> Self::V;
244    fn subus16(self, a: Self::V, b: Self::V) -> Self::V;
245    fn mullo16(self, a: Self::V, b: Self::V) -> Self::V;
246    fn mulhiu16(self, a: Self::V, b: Self::V) -> Self::V;
247    fn mins16(self, a: Self::V, b: Self::V) -> Self::V;
248    fn maxs16(self, a: Self::V, b: Self::V) -> Self::V;
249    fn cmpeq16(self, a: Self::V, b: Self::V) -> Self::V;
250    fn cmpgts16(self, a: Self::V, b: Self::V) -> Self::V;
251    fn srl16<const N: i32>(self, a: Self::V) -> Self::V;
252    fn sll16<const N: i32>(self, a: Self::V) -> Self::V;
253    /// pshuflw + pshufhw with the same immediate (per 64-bit half)
254    fn shuf16<const IMM: i32>(self, a: Self::V) -> Self::V;
255    fn unpacklo16(self, a: Self::V, b: Self::V) -> Self::V;
256    fn unpackhi16(self, a: Self::V, b: Self::V) -> Self::V;
257    fn packs32(self, a: Self::V, b: Self::V) -> Self::V;
258    /// pmaddwd: i32 lanes = a0 * b0 + a1 * b1 of the signed 16-bit pairs
259    fn madd16(self, a: Self::V, b: Self::V) -> Self::V;
260
261    // ---- 32-bit lanes
262    fn add32(self, a: Self::V, b: Self::V) -> Self::V;
263    fn sub32(self, a: Self::V, b: Self::V) -> Self::V;
264    fn cmpeq32(self, a: Self::V, b: Self::V) -> Self::V;
265    fn cmpgts32(self, a: Self::V, b: Self::V) -> Self::V;
266    /// bit i = top bit of dword i (BYTES / 4 bits)
267    fn movemask32(self, a: Self::V) -> u32;
268    /// pshufd (per 128-bit lane)
269    fn shuf32<const IMM: i32>(self, a: Self::V) -> Self::V;
270    /// shufps on integer registers: dwords 0, 1 from a, 2, 3 from b (per 128-bit lane)
271    fn shufps<const IMM: i32>(self, a: Self::V, b: Self::V) -> Self::V;
272    fn srl32<const N: i32>(self, a: Self::V) -> Self::V;
273    fn sll32<const N: i32>(self, a: Self::V) -> Self::V;
274    fn sra32<const N: i32>(self, a: Self::V) -> Self::V;
275    /// pslldq / psrldq: byte shifts per 128-bit lane
276    fn bsll<const N: i32>(self, a: Self::V) -> Self::V;
277    fn bsrl<const N: i32>(self, a: Self::V) -> Self::V;
278
279    // ---- bits
280    fn and(self, a: Self::V, b: Self::V) -> Self::V;
281    /// !a & b (pandn)
282    fn andnot(self, a: Self::V, b: Self::V) -> Self::V;
283    fn or(self, a: Self::V, b: Self::V) -> Self::V;
284    fn xor(self, a: Self::V, b: Self::V) -> Self::V;
285
286    // ---- f32 lanes
287    fn cvt_i32_f32(self, a: Self::V) -> Self::V;
288    /// truncating (0x80000000 for NaN / out of range)
289    fn cvtt_f32_i32(self, a: Self::V) -> Self::V;
290    /// rounding to nearest even (cvtps2dq)
291    fn cvtr_f32_i32(self, a: Self::V) -> Self::V;
292    fn addf(self, a: Self::V, b: Self::V) -> Self::V;
293    fn subf(self, a: Self::V, b: Self::V) -> Self::V;
294    fn mulf(self, a: Self::V, b: Self::V) -> Self::V;
295    fn divf(self, a: Self::V, b: Self::V) -> Self::V;
296
297    // ---- f64 lanes (BYTES / 8 of them); bit operations: and / andnot / or / xor
298    fn splatd(self, v: f64) -> Self::V;
299    fn addd(self, a: Self::V, b: Self::V) -> Self::V;
300    fn subd(self, a: Self::V, b: Self::V) -> Self::V;
301    fn muld(self, a: Self::V, b: Self::V) -> Self::V;
302    fn divd(self, a: Self::V, b: Self::V) -> Self::V;
303    fn sqrtd(self, a: Self::V) -> Self::V;
304    /// maxpd / minpd: b when unordered
305    fn maxd(self, a: Self::V, b: Self::V) -> Self::V;
306    fn mind(self, a: Self::V, b: Self::V) -> Self::V;
307    /// cmpltpd: all ones where a < b (false when unordered)
308    fn cmpltd(self, a: Self::V, b: Self::V) -> Self::V;
309    /// cvtdq2pd: the first BYTES / 8 i32 lanes to f64
310    fn cvt_i32_f64(self, a: Self::V) -> Self::V;
311    /// cvttpd2dq: f64 lanes to the first BYTES / 8 i32 lanes (truncating, 0x80000000 when
312    /// out of range), the rest 0
313    fn cvtt_f64_i32(self, a: Self::V) -> Self::V;
314    /// the first BYTES / 8 i32 lanes
315    fn store_i32_half(self, b: &mut [i32], v: Self::V);
316
317    // R6-jpeg, R6-xf: the JPEG kernels (img_jpeg dsp)
318    /// psraw: arithmetic right shift of the i16 lanes
319    fn sra16<const N: i32>(self, a: Self::V) -> Self::V;
320    /// pmulhw: high 16 bits of the signed 16-bit products
321    fn mulhis16(self, a: Self::V, b: Self::V) -> Self::V;
322    /// packsswb: i16 lanes of a, then b, saturated to i8
323    fn packs16(self, a: Self::V, b: Self::V) -> Self::V;
324    /// BYTES / 2 i16 values
325    fn load_i16(self, b: &[i16]) -> Self::V;
326    fn store_i16(self, b: &mut [i16], v: Self::V);
327    /// 2 * BYTES bytes: per 128-bit lane k, lane k of lo then lane k of hi (SSE2: lo, hi;
328    /// AVX2: vperm2i128) -- 2 x 4 pixels per lane in pixel order
329    fn store_lanes2(self, b: &mut [u8], lo: Self::V, hi: Self::V);
330    /// the inverse of store_lanes2: 2 * BYTES bytes -> (lo, hi)
331    fn load_lanes2(self, b: &[u8]) -> (Self::V, Self::V);
332    /// 8 i16 per 128-bit lane from separate sources: lane 0 from a, lane 1 from b (SSE2: a)
333    fn load_i16_lanes(self, a: &[i16], b: &[i16]) -> Self::V;
334    /// the low 8 bytes of every 128-bit lane to separate places: lane 0 to a, lane 1 to b
335    /// (SSE2: a)
336    fn store_raw_lanes(self, a: &mut [u8], b: &mut [u8], v: Self::V);
337    /// (main agent) psadbw: per 8-byte group the sum of |a - b| (u8) in its u64 lane
338    fn sadu8(self, a: Self::V, b: Self::V) -> Self::V;
339    /// (main agent) u64 lanes
340    fn add64(self, a: Self::V, b: Self::V) -> Self::V;
341
342    // R6-icc: the ICC transform kernels (img_core icc_xform)
343    /// maxps / minps: b when unordered
344    fn maxf(self, a: Self::V, b: Self::V) -> Self::V;
345    fn minf(self, a: Self::V, b: Self::V) -> Self::V;
346    /// sqrtps
347    fn sqrtf(self, a: Self::V) -> Self::V;
348
349    // r9-av1: the CDEF kernel (img_av1 cdef)
350    /// BYTES / 2 u16 values
351    fn load_u16(self, b: &[u16]) -> Self::V;
352    fn store_u16(self, b: &mut [u16], v: Self::V);
353    /// movq: 4 u16 values into the low 64 bits (the rest 0) / the low 64 bits to 4 u16 values
354    fn load_u16_lo(self, b: &[u16]) -> Self::V;
355    fn store_u16_lo(self, b: &mut [u16], v: Self::V);
356    /// psrlw with a register count: logical right shift of the 16-bit lanes by n (n > 15: 0)
357    fn srl16x(self, a: Self::V, n: u32) -> Self::V;
358
359    // R9-jxl: the JPEG XL render kernels (img_jxl)
360    /// BYTES / 4 f32 values
361    fn load_f32(self, b: &[f32]) -> Self::V;
362    fn store_f32(self, b: &mut [f32], v: Self::V);
363
364    // R9-webp: the VP8 loop filter (img_webp)
365    /// paddsb: signed saturating add of the i8 lanes
366    fn adds8(self, a: Self::V, b: Self::V) -> Self::V;
367    /// psubsb: signed saturating subtract of the i8 lanes
368    fn subs8(self, a: Self::V, b: Self::V) -> Self::V;
369
370    // r9-dds: the BC encoders' pixel-set helpers (img_dds enc)
371    /// cmpltps: all ones where a < b (false when unordered)
372    fn cmpltf(self, a: Self::V, b: Self::V) -> Self::V;
373
374    // jxl-speed: the JPEG XL writer's transforms (img_jxl dctk: 4 x 4 transposes per lane)
375    /// 4 f32 per 128-bit lane from separate sources: lane 0 from a, lane 1 from b (SSE2: a)
376    fn load_f32_lanes(self, a: &[f32], b: &[f32]) -> Self::V;
377    /// the 4 f32 of every 128-bit lane to separate places: lane 0 to a, lane 1 to b (SSE2: a)
378    fn store_f32_lanes(self, a: &mut [f32], b: &mut [f32], v: Self::V);
379
380    // perf-blend: the blend-mode kernels (gfx blendmode).  core::arch writes the pack
381    // intrinsics as generic smax / smin + shuffle; with AVX2 (SSE4.1's pmaxsd / pminsd
382    // available) LLVM keeps the clamps in front of the pack: 4 (packs32) / 2 (packus16) more
383    // instructions per pack.  On AVX2 these two are exactly one pack instruction (inline
384    // assembly); SSE2 uses the intrinsics (packs32 is one instruction there, packus16 keeps
385    // pmaxsw / pminsw: legacy-encoded inline assembly would hurt AVX2 code calling Sse2).
386    /// packs32 (packssdw), on AVX2 without the clamps
387    fn packssdw(self, a: Self::V, b: Self::V) -> Self::V;
388    /// pixel lanes to raw pixels: packuswb (+ vpermq 0x08 on AVX2) as in store_px, on AVX2
389    /// without the clamps; the inverse of widen_px
390    fn packuswb_px(self, v: Self::V) -> Self::V;
391    /// ptest on the raw-pixel bytes (SSE2: the low 8, AVX2: the low 16): 1 when a & b == 0
392    /// (ZF), else 2 when !a & b == 0 (CF), else 0.  AVX2: vptest + jz + jnc, the CF case
393    /// falls through (inline assembly: core::arch writes ptest as or-reductions, and LLVM
394    /// then often builds shuffle / reduce sequences or loses the CF test); branch on the
395    /// result, the branches thread through
396    fn ptest_raw(self, a: Self::V, b: Self::V) -> u32;
397    /// ptest ZF on the raw-pixel bytes: a & b == 0 (AVX2: vptest + jz)
398    fn ptestz_raw(self, a: Self::V, b: Self::V) -> bool;
399}