Skip to main content

simd/
lib.rs

1// simd -- SSE2 / AVX2 registers behind one safe interface, so a kernel is written once and
2// runs on both instruction sets with bit-identical results (the assembly's BMV = 0 / 1
3// macros); ARM64 and the browser in place of Sse2: neon.rs, wasm.rs
4//
5//   trait Simd       a register type V and its operations, named after the SSE2 instructions;
6//                    Sse2 (128-bit registers) and Avx2 (256-bit registers used as two
7//                    independent 128-bit lanes: every operation works per 128-bit lane, so
8//                    lane k computes exactly what SSE2 computes for its data)
9//   trait Kernel     a computation generic over Simd (run::<S>)
10//   dispatch(k)      runs k with Avx2 when rt::cpu reports CPU_AVX2, else with Sse2
11//   dispatch_sse2(k) always Sse2 (tests)
12//   has_avx2()
13//   copy_stream(dst, src) / stream_fence()   a copy whose stores bypass the caches (x86-64:
14//                    movntdq), for big results another thread reads later; the fence makes
15//                    them visible (the copying thread calls it before handing dst on)
16//
17// Pixel helpers work on "pixel lanes": the 4 bytes of a pixel zero-extended to 4 u16 lanes,
18// PX = 2 (SSE2) / 4 (AVX2) pixels per register, pixels 2k and 2k + 1 in 128-bit lane k.
19//
20// One of the four crates that may contain `unsafe` (rt, simd, plat, plugin).  Every
21// intrinsic is an unsafe call (core::arch requires the feature on the calling function, even
22// SSE2, the x86-64 baseline: sse!); Avx2 intrinsics are only reached through an Avx2 value,
23// which exists only after the CPU check (Avx2::new) -- the precondition of all of them.
24// Memory helpers check the slice lengths (panic, never an access outside the slice).
25// Kernels must not put closures or function pointers between dispatch and the intrinsics
26// (compiled outside the AVX2 context, every intrinsic would become a call); build_rs.py
27// refuses compute crates with out-of-line core_arch functions.
28#![no_std]
29
30// in a module's build (--cfg jflib_module, docs/CRATES.md) the module runtime goes by the same name
31#[cfg(jflib_module)]
32extern crate modrt as rt;
33
34#[cfg(target_arch = "x86_64")]
35mod native;
36#[cfg(target_arch = "x86_64")]
37pub use native::{Sse2, Avx2};
38mod scalar;
39pub use scalar::{Scalar, Scalar256};
40#[cfg(target_arch = "aarch64")] mod neon;
41#[cfg(target_arch = "aarch64")] pub use neon::{Neon128, Neon128 as Sse2};
42#[cfg(all(target_arch = "wasm32", target_feature = "simd128"))] mod wasm;
43#[cfg(all(target_arch = "wasm32", target_feature = "simd128"))] pub use wasm::{Wasm128, Wasm128 as Sse2};
44#[cfg(any(target_arch = "aarch64", all(target_arch = "wasm32", target_feature = "simd128")))]
45pub use scalar::Scalar256 as Avx2;
46#[cfg(not(any(target_arch = "x86_64", target_arch = "aarch64", all(target_arch = "wasm32", target_feature = "simd128"))))]
47pub use scalar::{Scalar as Sse2, Scalar256 as Avx2};
48
49pub use rt::cpu::CPU_AVX2;
50
51/// AVX2 kernels may be used (CPU and OS support, not disabled by `--simd sse2`)
52#[inline]
53pub fn has_avx2() -> bool {
54    rt::cpu::features() & CPU_AVX2 != 0
55}
56
57/// copy src into dst (the shorter length) with non-temporal stores where the target has them
58/// (x86-64: the 16-byte aligned part of dst): dst is neither read first nor pulled into the
59/// caches, which keep what the thread works on.  stream_fence() before dst is handed on.
60#[cfg(target_arch = "x86_64")]
61pub fn copy_stream(dst: &mut [u8], src: &[u8]) {
62    use core::arch::x86_64::{__m128i, _mm_loadu_si128, _mm_stream_si128};
63    let n = dst.len().min(src.len());
64    let (dst, src) = (&mut dst[..n], &src[..n]);
65    let head = (dst.as_ptr() as usize).wrapping_neg() & 15;
66    if n < head + 64 {
67        dst.copy_from_slice(src);
68        return;
69    }
70    dst[..head].copy_from_slice(&src[..head]);
71    let body = (n - head) & !63;
72    let (d, s) = (&mut dst[head..head + body], &src[head..head + body]);
73    let mut i = 0;
74    while i < body {
75        // SAFETY: i + 64 <= body: 64 readable bytes of s and 64 writable bytes of d at i; d's
76        // start is 16-byte aligned (head), so every store address is (movntdq's alignment).
77        unsafe {
78            let (sp, dp) = (s.as_ptr().add(i) as *const __m128i, d.as_mut_ptr().add(i) as *mut __m128i);
79            let (a, b, c, e) = (_mm_loadu_si128(sp), _mm_loadu_si128(sp.add(1)), _mm_loadu_si128(sp.add(2)), _mm_loadu_si128(sp.add(3)));
80            _mm_stream_si128(dp, a);
81            _mm_stream_si128(dp.add(1), b);
82            _mm_stream_si128(dp.add(2), c);
83            _mm_stream_si128(dp.add(3), e);
84        }
85        i += 64;
86    }
87    dst[head + body..].copy_from_slice(&src[head + body..]);
88}
89
90/// copy_stream without streaming stores (other targets).  ARM64 needs nothing of its own: stnp
91/// is only a hint and its loose ordering would want a barrier, while a copy like this one
92/// writes whole lines in a row, which ARM cores notice and then stop allocating in their caches.
93#[cfg(not(target_arch = "x86_64"))]
94pub fn copy_stream(dst: &mut [u8], src: &[u8]) {
95    let n = dst.len().min(src.len());
96    dst[..n].copy_from_slice(&src[..n]);
97}
98
99/// the streaming stores of this thread become visible to others (sfence); nothing to do on other
100/// targets: without such stores the release of whatever hands dst on is enough
101#[inline]
102pub fn stream_fence() {
103    // SAFETY: sfence is part of the x86-64 baseline (SSE); it only orders stores.
104    #[cfg(target_arch = "x86_64")]
105    unsafe {
106        core::arch::x86_64::_mm_sfence()
107    };
108}
109
110/// a computation written once for every instruction set
111pub trait Kernel {
112    type Out;
113    fn run<S: Simd>(self, s: S) -> Self::Out;
114}
115
116/// run k with the best instruction set of this CPU
117#[inline]
118#[cfg(target_arch = "x86_64")]
119pub fn dispatch<K: Kernel>(k: K) -> K::Out {
120    match Avx2::new() {
121        // SAFETY: Avx2::new succeeded, so the CPU and the OS support AVX2.
122        Some(a) => unsafe { run_avx2(k, a) },
123        None => k.run(Sse2),
124    }
125}
126
127/// run k with SSE2 (the baseline; tests compare it with dispatch)
128#[inline]
129pub fn dispatch_sse2<K: Kernel>(k: K) -> K::Out {
130    k.run(Sse2)
131}
132
133/// Run a kernel with NEON on ARM64, with SIMD128 when enabled by the WASM bundle, otherwise
134/// scalar registers.
135#[cfg(not(target_arch = "x86_64"))]
136#[inline]
137pub fn dispatch<K: Kernel>(k: K) -> K::Out { k.run(Sse2) }
138
139// the kernel compiled with AVX2 enabled (its #[inline(always)] code is inlined here)
140#[cfg(target_arch = "x86_64")]
141#[target_feature(enable = "avx2")]
142fn run_avx2<K: Kernel>(k: K, a: Avx2) -> K::Out {
143    k.run(a)
144}
145
146/// operations of one register type; integer lanes are named by their width (8 / 16 / 32),
147/// signed operations end in s, unsigned in u
148pub trait Simd: Copy {
149    type V: Copy;
150    /// bytes per register
151    const BYTES: usize;
152    /// pixels per register in pixel lanes
153    const PX: usize;
154
155    fn zero(self) -> Self::V;
156    fn splat8(self, v: u8) -> Self::V;
157    fn splat16(self, v: u16) -> Self::V;
158    fn splat32(self, v: u32) -> Self::V;
159    fn splat64(self, v: u64) -> Self::V;
160    fn splatf(self, v: f32) -> Self::V;
161
162    // ---- memory
163    /// BYTES bytes
164    fn load(self, b: &[u8]) -> Self::V;
165    fn store(self, b: &mut [u8], v: Self::V);
166    /// 4 * PX bytes (PX pixels) zero-extended to pixel lanes
167    fn load_px(self, b: &[u8]) -> Self::V;
168    /// pixel lanes saturated to bytes (packuswb), 4 * PX bytes
169    fn store_px(self, b: &mut [u8], v: Self::V);
170    /// PX bytes, byte i to the 4 u16 lanes of pixel i
171    fn bcast_px(self, b: &[u8]) -> Self::V;
172    /// 4 * PX bytes as they are ("raw" pixels: SSE2 the low 8 bytes, AVX2 the low 16)
173    fn load_raw(self, b: &[u8]) -> Self::V;
174    fn store_raw(self, b: &mut [u8], v: Self::V);
175    /// raw pixels to pixel lanes
176    fn widen_px(self, v: Self::V) -> Self::V;
177    /// 4 bytes into the low dword of every 128-bit lane (BYTES / 4 bytes), the rest 0
178    fn load_lane32(self, b: &[u8]) -> Self::V;
179    /// BYTES / 4 i32 values
180    fn load_i32(self, b: &[i32]) -> Self::V;
181    fn store_i32(self, b: &mut [i32], v: Self::V);
182    /// movd: v in dword 0, the rest 0 / dword 0 of v
183    fn from_i32(self, v: i32) -> Self::V;
184    fn to_i32(self, v: Self::V) -> i32;
185
186    // ---- 8-bit lanes
187    fn add8(self, a: Self::V, b: Self::V) -> Self::V;
188    fn sub8(self, a: Self::V, b: Self::V) -> Self::V;
189    fn addus8(self, a: Self::V, b: Self::V) -> Self::V;
190    fn subus8(self, a: Self::V, b: Self::V) -> Self::V;
191    fn maxu8(self, a: Self::V, b: Self::V) -> Self::V;
192    fn minu8(self, a: Self::V, b: Self::V) -> Self::V;
193    fn cmpeq8(self, a: Self::V, b: Self::V) -> Self::V;
194    fn avgu8(self, a: Self::V, b: Self::V) -> Self::V;
195    /// bit i = top bit of byte i (BYTES bits)
196    fn movemask8(self, a: Self::V) -> u32;
197    fn unpacklo8(self, a: Self::V, b: Self::V) -> Self::V;
198    fn unpackhi8(self, a: Self::V, b: Self::V) -> Self::V;
199    fn packus16(self, a: Self::V, b: Self::V) -> Self::V;
200
201    // ---- 16-bit lanes
202    fn add16(self, a: Self::V, b: Self::V) -> Self::V;
203    fn sub16(self, a: Self::V, b: Self::V) -> Self::V;
204    fn subus16(self, a: Self::V, b: Self::V) -> Self::V;
205    fn mullo16(self, a: Self::V, b: Self::V) -> Self::V;
206    fn mulhiu16(self, a: Self::V, b: Self::V) -> Self::V;
207    fn mins16(self, a: Self::V, b: Self::V) -> Self::V;
208    fn maxs16(self, a: Self::V, b: Self::V) -> Self::V;
209    fn cmpeq16(self, a: Self::V, b: Self::V) -> Self::V;
210    fn cmpgts16(self, a: Self::V, b: Self::V) -> Self::V;
211    fn srl16<const N: i32>(self, a: Self::V) -> Self::V;
212    fn sll16<const N: i32>(self, a: Self::V) -> Self::V;
213    /// pshuflw + pshufhw with the same immediate (per 64-bit half)
214    fn shuf16<const IMM: i32>(self, a: Self::V) -> Self::V;
215    fn unpacklo16(self, a: Self::V, b: Self::V) -> Self::V;
216    fn unpackhi16(self, a: Self::V, b: Self::V) -> Self::V;
217    fn packs32(self, a: Self::V, b: Self::V) -> Self::V;
218    /// pmaddwd: i32 lanes = a0 * b0 + a1 * b1 of the signed 16-bit pairs
219    fn madd16(self, a: Self::V, b: Self::V) -> Self::V;
220
221    // ---- 32-bit lanes
222    fn add32(self, a: Self::V, b: Self::V) -> Self::V;
223    fn sub32(self, a: Self::V, b: Self::V) -> Self::V;
224    fn cmpeq32(self, a: Self::V, b: Self::V) -> Self::V;
225    fn cmpgts32(self, a: Self::V, b: Self::V) -> Self::V;
226    /// bit i = top bit of dword i (BYTES / 4 bits)
227    fn movemask32(self, a: Self::V) -> u32;
228    /// pshufd (per 128-bit lane)
229    fn shuf32<const IMM: i32>(self, a: Self::V) -> Self::V;
230    /// shufps on integer registers: dwords 0, 1 from a, 2, 3 from b (per 128-bit lane)
231    fn shufps<const IMM: i32>(self, a: Self::V, b: Self::V) -> Self::V;
232    fn srl32<const N: i32>(self, a: Self::V) -> Self::V;
233    fn sll32<const N: i32>(self, a: Self::V) -> Self::V;
234    fn sra32<const N: i32>(self, a: Self::V) -> Self::V;
235    /// pslldq / psrldq: byte shifts per 128-bit lane
236    fn bsll<const N: i32>(self, a: Self::V) -> Self::V;
237    fn bsrl<const N: i32>(self, a: Self::V) -> Self::V;
238
239    // ---- bits
240    fn and(self, a: Self::V, b: Self::V) -> Self::V;
241    /// !a & b (pandn)
242    fn andnot(self, a: Self::V, b: Self::V) -> Self::V;
243    fn or(self, a: Self::V, b: Self::V) -> Self::V;
244    fn xor(self, a: Self::V, b: Self::V) -> Self::V;
245
246    // ---- f32 lanes
247    fn cvt_i32_f32(self, a: Self::V) -> Self::V;
248    /// truncating (0x80000000 for NaN / out of range)
249    fn cvtt_f32_i32(self, a: Self::V) -> Self::V;
250    /// rounding to nearest even (cvtps2dq)
251    fn cvtr_f32_i32(self, a: Self::V) -> Self::V;
252    fn addf(self, a: Self::V, b: Self::V) -> Self::V;
253    fn subf(self, a: Self::V, b: Self::V) -> Self::V;
254    fn mulf(self, a: Self::V, b: Self::V) -> Self::V;
255    fn divf(self, a: Self::V, b: Self::V) -> Self::V;
256
257    // ---- f64 lanes (BYTES / 8 of them); bit operations: and / andnot / or / xor
258    fn splatd(self, v: f64) -> Self::V;
259    fn addd(self, a: Self::V, b: Self::V) -> Self::V;
260    fn subd(self, a: Self::V, b: Self::V) -> Self::V;
261    fn muld(self, a: Self::V, b: Self::V) -> Self::V;
262    fn divd(self, a: Self::V, b: Self::V) -> Self::V;
263    fn sqrtd(self, a: Self::V) -> Self::V;
264    /// maxpd / minpd: b when unordered
265    fn maxd(self, a: Self::V, b: Self::V) -> Self::V;
266    fn mind(self, a: Self::V, b: Self::V) -> Self::V;
267    /// cmpltpd: all ones where a < b (false when unordered)
268    fn cmpltd(self, a: Self::V, b: Self::V) -> Self::V;
269    /// cvtdq2pd: the first BYTES / 8 i32 lanes to f64
270    fn cvt_i32_f64(self, a: Self::V) -> Self::V;
271    /// cvttpd2dq: f64 lanes to the first BYTES / 8 i32 lanes (truncating, 0x80000000 when
272    /// out of range), the rest 0
273    fn cvtt_f64_i32(self, a: Self::V) -> Self::V;
274    /// the first BYTES / 8 i32 lanes
275    fn store_i32_half(self, b: &mut [i32], v: Self::V);
276
277    // R6-jpeg, R6-xf: the JPEG kernels (img_jpeg dsp)
278    /// psraw: arithmetic right shift of the i16 lanes
279    fn sra16<const N: i32>(self, a: Self::V) -> Self::V;
280    /// pmulhw: high 16 bits of the signed 16-bit products
281    fn mulhis16(self, a: Self::V, b: Self::V) -> Self::V;
282    /// packsswb: i16 lanes of a, then b, saturated to i8
283    fn packs16(self, a: Self::V, b: Self::V) -> Self::V;
284    /// BYTES / 2 i16 values
285    fn load_i16(self, b: &[i16]) -> Self::V;
286    fn store_i16(self, b: &mut [i16], v: Self::V);
287    /// 2 * BYTES bytes: per 128-bit lane k, lane k of lo then lane k of hi (SSE2: lo, hi;
288    /// AVX2: vperm2i128) -- 2 x 4 pixels per lane in pixel order
289    fn store_lanes2(self, b: &mut [u8], lo: Self::V, hi: Self::V);
290    /// the inverse of store_lanes2: 2 * BYTES bytes -> (lo, hi)
291    fn load_lanes2(self, b: &[u8]) -> (Self::V, Self::V);
292    /// 8 i16 per 128-bit lane from separate sources: lane 0 from a, lane 1 from b (SSE2: a)
293    fn load_i16_lanes(self, a: &[i16], b: &[i16]) -> Self::V;
294    /// the low 8 bytes of every 128-bit lane to separate places: lane 0 to a, lane 1 to b
295    /// (SSE2: a)
296    fn store_raw_lanes(self, a: &mut [u8], b: &mut [u8], v: Self::V);
297    /// (main agent) psadbw: per 8-byte group the sum of |a - b| (u8) in its u64 lane
298    fn sadu8(self, a: Self::V, b: Self::V) -> Self::V;
299    /// (main agent) u64 lanes
300    fn add64(self, a: Self::V, b: Self::V) -> Self::V;
301
302    // R6-icc: the ICC transform kernels (img_core icc_xform)
303    /// maxps / minps: b when unordered
304    fn maxf(self, a: Self::V, b: Self::V) -> Self::V;
305    fn minf(self, a: Self::V, b: Self::V) -> Self::V;
306    /// sqrtps
307    fn sqrtf(self, a: Self::V) -> Self::V;
308
309    // r9-av1: the CDEF kernel (img_av1 cdef)
310    /// BYTES / 2 u16 values
311    fn load_u16(self, b: &[u16]) -> Self::V;
312    fn store_u16(self, b: &mut [u16], v: Self::V);
313    /// movq: 4 u16 values into the low 64 bits (the rest 0) / the low 64 bits to 4 u16 values
314    fn load_u16_lo(self, b: &[u16]) -> Self::V;
315    fn store_u16_lo(self, b: &mut [u16], v: Self::V);
316    /// psrlw with a register count: logical right shift of the 16-bit lanes by n (n > 15: 0)
317    fn srl16x(self, a: Self::V, n: u32) -> Self::V;
318
319    // R9-jxl: the JPEG XL render kernels (img_jxl)
320    /// BYTES / 4 f32 values
321    fn load_f32(self, b: &[f32]) -> Self::V;
322    fn store_f32(self, b: &mut [f32], v: Self::V);
323
324    // R9-webp: the VP8 loop filter (img_webp)
325    /// paddsb: signed saturating add of the i8 lanes
326    fn adds8(self, a: Self::V, b: Self::V) -> Self::V;
327    /// psubsb: signed saturating subtract of the i8 lanes
328    fn subs8(self, a: Self::V, b: Self::V) -> Self::V;
329
330    // r9-dds: the BC encoders' pixel-set helpers (img_dds enc)
331    /// cmpltps: all ones where a < b (false when unordered)
332    fn cmpltf(self, a: Self::V, b: Self::V) -> Self::V;
333
334    // jxl-speed: the JPEG XL writer's transforms (img_jxl dctk: 4 x 4 transposes per lane)
335    /// 4 f32 per 128-bit lane from separate sources: lane 0 from a, lane 1 from b (SSE2: a)
336    fn load_f32_lanes(self, a: &[f32], b: &[f32]) -> Self::V;
337    /// the 4 f32 of every 128-bit lane to separate places: lane 0 to a, lane 1 to b (SSE2: a)
338    fn store_f32_lanes(self, a: &mut [f32], b: &mut [f32], v: Self::V);
339
340    // perf-blend: the blend-mode kernels (gfx blendmode).  core::arch writes the pack
341    // intrinsics as generic smax / smin + shuffle; with AVX2 (SSE4.1's pmaxsd / pminsd
342    // available) LLVM keeps the clamps in front of the pack: 4 (packs32) / 2 (packus16) more
343    // instructions per pack.  On AVX2 these two are exactly one pack instruction (inline
344    // assembly); SSE2 uses the intrinsics (packs32 is one instruction there, packus16 keeps
345    // pmaxsw / pminsw: legacy-encoded inline assembly would hurt AVX2 code calling Sse2).
346    /// packs32 (packssdw), on AVX2 without the clamps
347    fn packssdw(self, a: Self::V, b: Self::V) -> Self::V;
348    /// pixel lanes to raw pixels: packuswb (+ vpermq 0x08 on AVX2) as in store_px, on AVX2
349    /// without the clamps; the inverse of widen_px
350    fn packuswb_px(self, v: Self::V) -> Self::V;
351    /// ptest on the raw-pixel bytes (SSE2: the low 8, AVX2: the low 16): 1 when a & b == 0
352    /// (ZF), else 2 when !a & b == 0 (CF), else 0.  AVX2: vptest + jz + jnc, the CF case
353    /// falls through (inline assembly: core::arch writes ptest as or-reductions, and LLVM
354    /// then often builds shuffle / reduce sequences or loses the CF test); branch on the
355    /// result, the branches thread through
356    fn ptest_raw(self, a: Self::V, b: Self::V) -> u32;
357    /// ptest ZF on the raw-pixel bytes: a & b == 0 (AVX2: vptest + jz)
358    fn ptestz_raw(self, a: Self::V, b: Self::V) -> bool;
359}