simd/lib.rs
1// simd -- SSE2 / AVX2 registers behind one safe interface, so a kernel is written once and
2// runs on both instruction sets with bit-identical results (the assembly's BMV = 0 / 1
3// macros); ARM64 and the browser in place of Sse2: neon.rs, wasm.rs
4//
5// trait Simd a register type V and its operations, named after the SSE2 instructions;
6// Sse2 (128-bit registers) and Avx2 (256-bit registers used as two
7// independent 128-bit lanes: every operation works per 128-bit lane, so
8// lane k computes exactly what SSE2 computes for its data)
9// trait Kernel a computation generic over Simd (run::<S>)
10// dispatch(k) runs k with Avx2 when rt::cpu reports CPU_AVX2, else with Sse2
11// dispatch_sse2(k) always Sse2 (tests)
12// has_avx2()
13// copy_stream(dst, src) / stream_fence() a copy whose stores bypass the caches (x86-64:
14// movntdq), for big results another thread reads later; the fence makes
15// them visible (the copying thread calls it before handing dst on)
16//
17// Pixel helpers work on "pixel lanes": the 4 bytes of a pixel zero-extended to 4 u16 lanes,
18// PX = 2 (SSE2) / 4 (AVX2) pixels per register, pixels 2k and 2k + 1 in 128-bit lane k.
19//
20// One of the four crates that may contain `unsafe` (rt, simd, plat, plugin). Every
21// intrinsic is an unsafe call (core::arch requires the feature on the calling function, even
22// SSE2, the x86-64 baseline: sse!); Avx2 intrinsics are only reached through an Avx2 value,
23// which exists only after the CPU check (Avx2::new) -- the precondition of all of them.
24// Memory helpers check the slice lengths (panic, never an access outside the slice).
25// Kernels must not put closures or function pointers between dispatch and the intrinsics
26// (compiled outside the AVX2 context, every intrinsic would become a call); build_rs.py
27// refuses compute crates with out-of-line core_arch functions.
28#![no_std]
29
30// in a module's build (--cfg jflib_module, docs/CRATES.md) the module runtime goes by the same name
31#[cfg(jflib_module)]
32extern crate modrt as rt;
33
34#[cfg(target_arch = "x86_64")]
35mod native;
36#[cfg(target_arch = "x86_64")]
37pub use native::{Sse2, Avx2};
38mod scalar;
39pub use scalar::{Scalar, Scalar256};
40#[cfg(target_arch = "aarch64")] mod neon;
41#[cfg(target_arch = "aarch64")] pub use neon::{Neon128, Neon128 as Sse2};
42#[cfg(all(target_arch = "wasm32", target_feature = "simd128"))] mod wasm;
43#[cfg(all(target_arch = "wasm32", target_feature = "simd128"))] pub use wasm::{Wasm128, Wasm128 as Sse2};
44#[cfg(any(target_arch = "aarch64", all(target_arch = "wasm32", target_feature = "simd128")))]
45pub use scalar::Scalar256 as Avx2;
46#[cfg(not(any(target_arch = "x86_64", target_arch = "aarch64", all(target_arch = "wasm32", target_feature = "simd128"))))]
47pub use scalar::{Scalar as Sse2, Scalar256 as Avx2};
48
49pub use rt::cpu::CPU_AVX2;
50
51/// AVX2 kernels may be used (CPU and OS support, not disabled by `--simd sse2`)
52#[inline]
53pub fn has_avx2() -> bool {
54 rt::cpu::features() & CPU_AVX2 != 0
55}
56
57/// copy src into dst (the shorter length) with non-temporal stores where the target has them
58/// (x86-64: the 16-byte aligned part of dst): dst is neither read first nor pulled into the
59/// caches, which keep what the thread works on. stream_fence() before dst is handed on.
60#[cfg(target_arch = "x86_64")]
61pub fn copy_stream(dst: &mut [u8], src: &[u8]) {
62 use core::arch::x86_64::{__m128i, _mm_loadu_si128, _mm_stream_si128};
63 let n = dst.len().min(src.len());
64 let (dst, src) = (&mut dst[..n], &src[..n]);
65 let head = (dst.as_ptr() as usize).wrapping_neg() & 15;
66 if n < head + 64 {
67 dst.copy_from_slice(src);
68 return;
69 }
70 dst[..head].copy_from_slice(&src[..head]);
71 let body = (n - head) & !63;
72 let (d, s) = (&mut dst[head..head + body], &src[head..head + body]);
73 let mut i = 0;
74 while i < body {
75 // SAFETY: i + 64 <= body: 64 readable bytes of s and 64 writable bytes of d at i; d's
76 // start is 16-byte aligned (head), so every store address is (movntdq's alignment).
77 unsafe {
78 let (sp, dp) = (s.as_ptr().add(i) as *const __m128i, d.as_mut_ptr().add(i) as *mut __m128i);
79 let (a, b, c, e) = (_mm_loadu_si128(sp), _mm_loadu_si128(sp.add(1)), _mm_loadu_si128(sp.add(2)), _mm_loadu_si128(sp.add(3)));
80 _mm_stream_si128(dp, a);
81 _mm_stream_si128(dp.add(1), b);
82 _mm_stream_si128(dp.add(2), c);
83 _mm_stream_si128(dp.add(3), e);
84 }
85 i += 64;
86 }
87 dst[head + body..].copy_from_slice(&src[head + body..]);
88}
89
90/// copy_stream without streaming stores (other targets). ARM64 needs nothing of its own: stnp
91/// is only a hint and its loose ordering would want a barrier, while a copy like this one
92/// writes whole lines in a row, which ARM cores notice and then stop allocating in their caches.
93#[cfg(not(target_arch = "x86_64"))]
94pub fn copy_stream(dst: &mut [u8], src: &[u8]) {
95 let n = dst.len().min(src.len());
96 dst[..n].copy_from_slice(&src[..n]);
97}
98
99/// the streaming stores of this thread become visible to others (sfence); nothing to do on other
100/// targets: without such stores the release of whatever hands dst on is enough
101#[inline]
102pub fn stream_fence() {
103 // SAFETY: sfence is part of the x86-64 baseline (SSE); it only orders stores.
104 #[cfg(target_arch = "x86_64")]
105 unsafe {
106 core::arch::x86_64::_mm_sfence()
107 };
108}
109
110/// a computation written once for every instruction set
111pub trait Kernel {
112 type Out;
113 fn run<S: Simd>(self, s: S) -> Self::Out;
114}
115
116/// run k with the best instruction set of this CPU
117#[inline]
118#[cfg(target_arch = "x86_64")]
119pub fn dispatch<K: Kernel>(k: K) -> K::Out {
120 match Avx2::new() {
121 // SAFETY: Avx2::new succeeded, so the CPU and the OS support AVX2.
122 Some(a) => unsafe { run_avx2(k, a) },
123 None => k.run(Sse2),
124 }
125}
126
127/// run k with SSE2 (the baseline; tests compare it with dispatch)
128#[inline]
129pub fn dispatch_sse2<K: Kernel>(k: K) -> K::Out {
130 k.run(Sse2)
131}
132
133/// Run a kernel with NEON on ARM64, with SIMD128 when enabled by the WASM bundle, otherwise
134/// scalar registers.
135#[cfg(not(target_arch = "x86_64"))]
136#[inline]
137pub fn dispatch<K: Kernel>(k: K) -> K::Out { k.run(Sse2) }
138
139// the kernel compiled with AVX2 enabled (its #[inline(always)] code is inlined here)
140#[cfg(target_arch = "x86_64")]
141#[target_feature(enable = "avx2")]
142fn run_avx2<K: Kernel>(k: K, a: Avx2) -> K::Out {
143 k.run(a)
144}
145
146/// operations of one register type; integer lanes are named by their width (8 / 16 / 32),
147/// signed operations end in s, unsigned in u
148pub trait Simd: Copy {
149 type V: Copy;
150 /// bytes per register
151 const BYTES: usize;
152 /// pixels per register in pixel lanes
153 const PX: usize;
154
155 fn zero(self) -> Self::V;
156 fn splat8(self, v: u8) -> Self::V;
157 fn splat16(self, v: u16) -> Self::V;
158 fn splat32(self, v: u32) -> Self::V;
159 fn splat64(self, v: u64) -> Self::V;
160 fn splatf(self, v: f32) -> Self::V;
161
162 // ---- memory
163 /// BYTES bytes
164 fn load(self, b: &[u8]) -> Self::V;
165 fn store(self, b: &mut [u8], v: Self::V);
166 /// 4 * PX bytes (PX pixels) zero-extended to pixel lanes
167 fn load_px(self, b: &[u8]) -> Self::V;
168 /// pixel lanes saturated to bytes (packuswb), 4 * PX bytes
169 fn store_px(self, b: &mut [u8], v: Self::V);
170 /// PX bytes, byte i to the 4 u16 lanes of pixel i
171 fn bcast_px(self, b: &[u8]) -> Self::V;
172 /// 4 * PX bytes as they are ("raw" pixels: SSE2 the low 8 bytes, AVX2 the low 16)
173 fn load_raw(self, b: &[u8]) -> Self::V;
174 fn store_raw(self, b: &mut [u8], v: Self::V);
175 /// raw pixels to pixel lanes
176 fn widen_px(self, v: Self::V) -> Self::V;
177 /// 4 bytes into the low dword of every 128-bit lane (BYTES / 4 bytes), the rest 0
178 fn load_lane32(self, b: &[u8]) -> Self::V;
179 /// BYTES / 4 i32 values
180 fn load_i32(self, b: &[i32]) -> Self::V;
181 fn store_i32(self, b: &mut [i32], v: Self::V);
182 /// movd: v in dword 0, the rest 0 / dword 0 of v
183 fn from_i32(self, v: i32) -> Self::V;
184 fn to_i32(self, v: Self::V) -> i32;
185
186 // ---- 8-bit lanes
187 fn add8(self, a: Self::V, b: Self::V) -> Self::V;
188 fn sub8(self, a: Self::V, b: Self::V) -> Self::V;
189 fn addus8(self, a: Self::V, b: Self::V) -> Self::V;
190 fn subus8(self, a: Self::V, b: Self::V) -> Self::V;
191 fn maxu8(self, a: Self::V, b: Self::V) -> Self::V;
192 fn minu8(self, a: Self::V, b: Self::V) -> Self::V;
193 fn cmpeq8(self, a: Self::V, b: Self::V) -> Self::V;
194 fn avgu8(self, a: Self::V, b: Self::V) -> Self::V;
195 /// bit i = top bit of byte i (BYTES bits)
196 fn movemask8(self, a: Self::V) -> u32;
197 fn unpacklo8(self, a: Self::V, b: Self::V) -> Self::V;
198 fn unpackhi8(self, a: Self::V, b: Self::V) -> Self::V;
199 fn packus16(self, a: Self::V, b: Self::V) -> Self::V;
200
201 // ---- 16-bit lanes
202 fn add16(self, a: Self::V, b: Self::V) -> Self::V;
203 fn sub16(self, a: Self::V, b: Self::V) -> Self::V;
204 fn subus16(self, a: Self::V, b: Self::V) -> Self::V;
205 fn mullo16(self, a: Self::V, b: Self::V) -> Self::V;
206 fn mulhiu16(self, a: Self::V, b: Self::V) -> Self::V;
207 fn mins16(self, a: Self::V, b: Self::V) -> Self::V;
208 fn maxs16(self, a: Self::V, b: Self::V) -> Self::V;
209 fn cmpeq16(self, a: Self::V, b: Self::V) -> Self::V;
210 fn cmpgts16(self, a: Self::V, b: Self::V) -> Self::V;
211 fn srl16<const N: i32>(self, a: Self::V) -> Self::V;
212 fn sll16<const N: i32>(self, a: Self::V) -> Self::V;
213 /// pshuflw + pshufhw with the same immediate (per 64-bit half)
214 fn shuf16<const IMM: i32>(self, a: Self::V) -> Self::V;
215 fn unpacklo16(self, a: Self::V, b: Self::V) -> Self::V;
216 fn unpackhi16(self, a: Self::V, b: Self::V) -> Self::V;
217 fn packs32(self, a: Self::V, b: Self::V) -> Self::V;
218 /// pmaddwd: i32 lanes = a0 * b0 + a1 * b1 of the signed 16-bit pairs
219 fn madd16(self, a: Self::V, b: Self::V) -> Self::V;
220
221 // ---- 32-bit lanes
222 fn add32(self, a: Self::V, b: Self::V) -> Self::V;
223 fn sub32(self, a: Self::V, b: Self::V) -> Self::V;
224 fn cmpeq32(self, a: Self::V, b: Self::V) -> Self::V;
225 fn cmpgts32(self, a: Self::V, b: Self::V) -> Self::V;
226 /// bit i = top bit of dword i (BYTES / 4 bits)
227 fn movemask32(self, a: Self::V) -> u32;
228 /// pshufd (per 128-bit lane)
229 fn shuf32<const IMM: i32>(self, a: Self::V) -> Self::V;
230 /// shufps on integer registers: dwords 0, 1 from a, 2, 3 from b (per 128-bit lane)
231 fn shufps<const IMM: i32>(self, a: Self::V, b: Self::V) -> Self::V;
232 fn srl32<const N: i32>(self, a: Self::V) -> Self::V;
233 fn sll32<const N: i32>(self, a: Self::V) -> Self::V;
234 fn sra32<const N: i32>(self, a: Self::V) -> Self::V;
235 /// pslldq / psrldq: byte shifts per 128-bit lane
236 fn bsll<const N: i32>(self, a: Self::V) -> Self::V;
237 fn bsrl<const N: i32>(self, a: Self::V) -> Self::V;
238
239 // ---- bits
240 fn and(self, a: Self::V, b: Self::V) -> Self::V;
241 /// !a & b (pandn)
242 fn andnot(self, a: Self::V, b: Self::V) -> Self::V;
243 fn or(self, a: Self::V, b: Self::V) -> Self::V;
244 fn xor(self, a: Self::V, b: Self::V) -> Self::V;
245
246 // ---- f32 lanes
247 fn cvt_i32_f32(self, a: Self::V) -> Self::V;
248 /// truncating (0x80000000 for NaN / out of range)
249 fn cvtt_f32_i32(self, a: Self::V) -> Self::V;
250 /// rounding to nearest even (cvtps2dq)
251 fn cvtr_f32_i32(self, a: Self::V) -> Self::V;
252 fn addf(self, a: Self::V, b: Self::V) -> Self::V;
253 fn subf(self, a: Self::V, b: Self::V) -> Self::V;
254 fn mulf(self, a: Self::V, b: Self::V) -> Self::V;
255 fn divf(self, a: Self::V, b: Self::V) -> Self::V;
256
257 // ---- f64 lanes (BYTES / 8 of them); bit operations: and / andnot / or / xor
258 fn splatd(self, v: f64) -> Self::V;
259 fn addd(self, a: Self::V, b: Self::V) -> Self::V;
260 fn subd(self, a: Self::V, b: Self::V) -> Self::V;
261 fn muld(self, a: Self::V, b: Self::V) -> Self::V;
262 fn divd(self, a: Self::V, b: Self::V) -> Self::V;
263 fn sqrtd(self, a: Self::V) -> Self::V;
264 /// maxpd / minpd: b when unordered
265 fn maxd(self, a: Self::V, b: Self::V) -> Self::V;
266 fn mind(self, a: Self::V, b: Self::V) -> Self::V;
267 /// cmpltpd: all ones where a < b (false when unordered)
268 fn cmpltd(self, a: Self::V, b: Self::V) -> Self::V;
269 /// cvtdq2pd: the first BYTES / 8 i32 lanes to f64
270 fn cvt_i32_f64(self, a: Self::V) -> Self::V;
271 /// cvttpd2dq: f64 lanes to the first BYTES / 8 i32 lanes (truncating, 0x80000000 when
272 /// out of range), the rest 0
273 fn cvtt_f64_i32(self, a: Self::V) -> Self::V;
274 /// the first BYTES / 8 i32 lanes
275 fn store_i32_half(self, b: &mut [i32], v: Self::V);
276
277 // R6-jpeg, R6-xf: the JPEG kernels (img_jpeg dsp)
278 /// psraw: arithmetic right shift of the i16 lanes
279 fn sra16<const N: i32>(self, a: Self::V) -> Self::V;
280 /// pmulhw: high 16 bits of the signed 16-bit products
281 fn mulhis16(self, a: Self::V, b: Self::V) -> Self::V;
282 /// packsswb: i16 lanes of a, then b, saturated to i8
283 fn packs16(self, a: Self::V, b: Self::V) -> Self::V;
284 /// BYTES / 2 i16 values
285 fn load_i16(self, b: &[i16]) -> Self::V;
286 fn store_i16(self, b: &mut [i16], v: Self::V);
287 /// 2 * BYTES bytes: per 128-bit lane k, lane k of lo then lane k of hi (SSE2: lo, hi;
288 /// AVX2: vperm2i128) -- 2 x 4 pixels per lane in pixel order
289 fn store_lanes2(self, b: &mut [u8], lo: Self::V, hi: Self::V);
290 /// the inverse of store_lanes2: 2 * BYTES bytes -> (lo, hi)
291 fn load_lanes2(self, b: &[u8]) -> (Self::V, Self::V);
292 /// 8 i16 per 128-bit lane from separate sources: lane 0 from a, lane 1 from b (SSE2: a)
293 fn load_i16_lanes(self, a: &[i16], b: &[i16]) -> Self::V;
294 /// the low 8 bytes of every 128-bit lane to separate places: lane 0 to a, lane 1 to b
295 /// (SSE2: a)
296 fn store_raw_lanes(self, a: &mut [u8], b: &mut [u8], v: Self::V);
297 /// (main agent) psadbw: per 8-byte group the sum of |a - b| (u8) in its u64 lane
298 fn sadu8(self, a: Self::V, b: Self::V) -> Self::V;
299 /// (main agent) u64 lanes
300 fn add64(self, a: Self::V, b: Self::V) -> Self::V;
301
302 // R6-icc: the ICC transform kernels (img_core icc_xform)
303 /// maxps / minps: b when unordered
304 fn maxf(self, a: Self::V, b: Self::V) -> Self::V;
305 fn minf(self, a: Self::V, b: Self::V) -> Self::V;
306 /// sqrtps
307 fn sqrtf(self, a: Self::V) -> Self::V;
308
309 // r9-av1: the CDEF kernel (img_av1 cdef)
310 /// BYTES / 2 u16 values
311 fn load_u16(self, b: &[u16]) -> Self::V;
312 fn store_u16(self, b: &mut [u16], v: Self::V);
313 /// movq: 4 u16 values into the low 64 bits (the rest 0) / the low 64 bits to 4 u16 values
314 fn load_u16_lo(self, b: &[u16]) -> Self::V;
315 fn store_u16_lo(self, b: &mut [u16], v: Self::V);
316 /// psrlw with a register count: logical right shift of the 16-bit lanes by n (n > 15: 0)
317 fn srl16x(self, a: Self::V, n: u32) -> Self::V;
318
319 // R9-jxl: the JPEG XL render kernels (img_jxl)
320 /// BYTES / 4 f32 values
321 fn load_f32(self, b: &[f32]) -> Self::V;
322 fn store_f32(self, b: &mut [f32], v: Self::V);
323
324 // R9-webp: the VP8 loop filter (img_webp)
325 /// paddsb: signed saturating add of the i8 lanes
326 fn adds8(self, a: Self::V, b: Self::V) -> Self::V;
327 /// psubsb: signed saturating subtract of the i8 lanes
328 fn subs8(self, a: Self::V, b: Self::V) -> Self::V;
329
330 // r9-dds: the BC encoders' pixel-set helpers (img_dds enc)
331 /// cmpltps: all ones where a < b (false when unordered)
332 fn cmpltf(self, a: Self::V, b: Self::V) -> Self::V;
333
334 // jxl-speed: the JPEG XL writer's transforms (img_jxl dctk: 4 x 4 transposes per lane)
335 /// 4 f32 per 128-bit lane from separate sources: lane 0 from a, lane 1 from b (SSE2: a)
336 fn load_f32_lanes(self, a: &[f32], b: &[f32]) -> Self::V;
337 /// the 4 f32 of every 128-bit lane to separate places: lane 0 to a, lane 1 to b (SSE2: a)
338 fn store_f32_lanes(self, a: &mut [f32], b: &mut [f32], v: Self::V);
339
340 // perf-blend: the blend-mode kernels (gfx blendmode). core::arch writes the pack
341 // intrinsics as generic smax / smin + shuffle; with AVX2 (SSE4.1's pmaxsd / pminsd
342 // available) LLVM keeps the clamps in front of the pack: 4 (packs32) / 2 (packus16) more
343 // instructions per pack. On AVX2 these two are exactly one pack instruction (inline
344 // assembly); SSE2 uses the intrinsics (packs32 is one instruction there, packus16 keeps
345 // pmaxsw / pminsw: legacy-encoded inline assembly would hurt AVX2 code calling Sse2).
346 /// packs32 (packssdw), on AVX2 without the clamps
347 fn packssdw(self, a: Self::V, b: Self::V) -> Self::V;
348 /// pixel lanes to raw pixels: packuswb (+ vpermq 0x08 on AVX2) as in store_px, on AVX2
349 /// without the clamps; the inverse of widen_px
350 fn packuswb_px(self, v: Self::V) -> Self::V;
351 /// ptest on the raw-pixel bytes (SSE2: the low 8, AVX2: the low 16): 1 when a & b == 0
352 /// (ZF), else 2 when !a & b == 0 (CF), else 0. AVX2: vptest + jz + jnc, the CF case
353 /// falls through (inline assembly: core::arch writes ptest as or-reductions, and LLVM
354 /// then often builds shuffle / reduce sequences or loses the CF test); branch on the
355 /// result, the branches thread through
356 fn ptest_raw(self, a: Self::V, b: Self::V) -> u32;
357 /// ptest ZF on the raw-pixel bytes: a & b == 0 (AVX2: vptest + jz)
358 fn ptestz_raw(self, a: Self::V, b: Self::V) -> bool;
359}