Skip to main content

simd/
scalar.rs

1// Portable scalar implementations of the complete Simd contract. Registers are
2// little-endian bytes; shuffles/packs work per 128-bit lane just like the native path.
3// Kept available on native targets so tests compare the actual implementations.
4
5use crate::Simd;
6#[cfg(target_arch = "wasm32")]
7use rt::fx as portable_fx;
8#[cfg(not(target_arch = "wasm32"))]
9use rt::portable::fx as portable_fx;
10
11/// Portable 128-bit register backend, the first WebAssembly baseline.
12#[derive(Clone, Copy)]
13pub struct Scalar;
14/// Portable pair of 128-bit lanes, useful for AVX2-equivalent conformance tests.
15#[derive(Clone, Copy)]
16pub struct Scalar256;
17impl Scalar256 {
18    /// AVX2 hardware is not provided by this compatibility type.
19    pub fn new() -> Option<Self> {
20        None
21    }
22}
23
24#[inline(always)]
25fn u16at(a: &[u8], i: usize) -> u16 {
26    u16::from_le_bytes([a[i], a[i + 1]])
27}
28#[inline(always)]
29fn u32at(a: &[u8], i: usize) -> u32 {
30    u32::from_le_bytes([a[i], a[i + 1], a[i + 2], a[i + 3]])
31}
32#[inline(always)]
33fn u64at(a: &[u8], i: usize) -> u64 {
34    u64::from_le_bytes([
35        a[i],
36        a[i + 1],
37        a[i + 2],
38        a[i + 3],
39        a[i + 4],
40        a[i + 5],
41        a[i + 6],
42        a[i + 7],
43    ])
44}
45
46macro_rules! backend {
47    ($name:ident, $n:expr) => {
48        impl Simd for $name {
49            type V = [u8; $n];
50            const BYTES: usize = $n;
51            const PX: usize = $n / 8;
52            #[inline(always)]
53            fn zero(self) -> Self::V {
54                [0; $n]
55            }
56            #[inline(always)]
57            fn splat8(self, v: u8) -> Self::V {
58                let mut out = [0u8; $n];
59                for i in (0..$n).step_by(1) {
60                    out[i] = v;
61                }
62                out
63            }
64            #[inline(always)]
65            fn splat16(self, v: u16) -> Self::V {
66                let mut out = [0u8; $n];
67                for i in (0..$n).step_by(2) {
68                    out[i..i + 2].copy_from_slice(&(v).to_le_bytes());
69                }
70                out
71            }
72            #[inline(always)]
73            fn splat32(self, v: u32) -> Self::V {
74                let mut out = [0u8; $n];
75                for i in (0..$n).step_by(4) {
76                    out[i..i + 4].copy_from_slice(&(v).to_le_bytes());
77                }
78                out
79            }
80            #[inline(always)]
81            fn splat64(self, v: u64) -> Self::V {
82                let mut out = [0u8; $n];
83                for i in (0..$n).step_by(8) {
84                    out[i..i + 8].copy_from_slice(&(v).to_le_bytes());
85                }
86                out
87            }
88            #[inline(always)]
89            fn splatf(self, v: f32) -> Self::V {
90                self.splat32(v.to_bits())
91            }
92            #[inline(always)]
93            fn splatd(self, v: f64) -> Self::V {
94                self.splat64(v.to_bits())
95            }
96            #[inline(always)]
97            fn load(self, b: &[u8]) -> Self::V {
98                let mut out = [0; $n];
99                out.copy_from_slice(&b[..$n]);
100                out
101            }
102            #[inline(always)]
103            fn store(self, b: &mut [u8], v: Self::V) -> () {
104                b[..$n].copy_from_slice(&v);
105            }
106            #[inline(always)]
107            fn load_raw(self, b: &[u8]) -> Self::V {
108                let mut out = [0; $n];
109                out[..$n / 2].copy_from_slice(&b[..$n / 2]);
110                out
111            }
112            #[inline(always)]
113            fn store_raw(self, b: &mut [u8], v: Self::V) -> () {
114                b[..$n / 2].copy_from_slice(&v[..$n / 2]);
115            }
116            #[inline(always)]
117            fn widen_px(self, v: Self::V) -> Self::V {
118                let mut out = [0; $n];
119                for i in 0..$n / 2 {
120                    out[i * 2] = v[i];
121                }
122                out
123            }
124            #[inline(always)]
125            fn load_px(self, b: &[u8]) -> Self::V {
126                self.widen_px(self.load_raw(b))
127            }
128            #[inline(always)]
129            fn store_px(self, b: &mut [u8], v: Self::V) -> () {
130                for i in 0..$n / 2 {
131                    b[i] = (u16at(&v, i * 2) as i16).clamp(0, 255) as u8;
132                }
133            }
134            #[inline(always)]
135            fn bcast_px(self, b: &[u8]) -> Self::V {
136                let mut out = [0; $n];
137                for i in 0..$n / 8 {
138                    for j in 0..4 {
139                        out[i * 8 + j * 2] = b[i];
140                    }
141                }
142                out
143            }
144            #[inline(always)]
145            fn load_lane32(self, b: &[u8]) -> Self::V {
146                let mut out = [0; $n];
147                for lane in 0..$n / 16 {
148                    out[lane * 16..lane * 16 + 4].copy_from_slice(&b[lane * 4..lane * 4 + 4]);
149                }
150                out
151            }
152            #[inline(always)]
153            fn from_i32(self, v: i32) -> Self::V {
154                let mut out = [0; $n];
155                out[..4].copy_from_slice(&v.to_le_bytes());
156                out
157            }
158            #[inline(always)]
159            fn to_i32(self, v: Self::V) -> i32 {
160                u32at(&v, 0) as i32
161            }
162            #[inline(always)]
163            fn load_i16(self, b: &[i16]) -> Self::V {
164                let mut out = [0; $n];
165                for i in 0..$n / 2 {
166                    out[i * 2..(i + 1) * 2].copy_from_slice(&(b[i]).to_le_bytes());
167                }
168                out
169            }
170            #[inline(always)]
171            fn store_i16(self, b: &mut [i16], v: Self::V) -> () {
172                for i in 0..$n / 2 {
173                    b[i] = u16at(&v, i * 2) as i16;
174                }
175            }
176            #[inline(always)]
177            fn load_u16(self, b: &[u16]) -> Self::V {
178                let mut out = [0; $n];
179                for i in 0..$n / 2 {
180                    out[i * 2..(i + 1) * 2].copy_from_slice(&(b[i]).to_le_bytes());
181                }
182                out
183            }
184            #[inline(always)]
185            fn store_u16(self, b: &mut [u16], v: Self::V) -> () {
186                for i in 0..$n / 2 {
187                    b[i] = u16at(&v, i * 2) as u16;
188                }
189            }
190            #[inline(always)]
191            fn load_i32(self, b: &[i32]) -> Self::V {
192                let mut out = [0; $n];
193                for i in 0..$n / 4 {
194                    out[i * 4..(i + 1) * 4].copy_from_slice(&(b[i]).to_le_bytes());
195                }
196                out
197            }
198            #[inline(always)]
199            fn store_i32(self, b: &mut [i32], v: Self::V) -> () {
200                for i in 0..$n / 4 {
201                    b[i] = u32at(&v, i * 4) as i32;
202                }
203            }
204            #[inline(always)]
205            fn load_f32(self, b: &[f32]) -> Self::V {
206                let mut out = [0; $n];
207                for i in 0..$n / 4 {
208                    out[i * 4..(i + 1) * 4].copy_from_slice(&(b[i].to_bits()).to_le_bytes());
209                }
210                out
211            }
212            #[inline(always)]
213            fn store_f32(self, b: &mut [f32], v: Self::V) -> () {
214                for i in 0..$n / 4 {
215                    b[i] = f32::from_bits(u32at(&v, i * 4));
216                }
217            }
218            #[inline(always)]
219            fn load_u16_lo(self, b: &[u16]) -> Self::V {
220                let mut out = [0; $n];
221                for i in 0..4 {
222                    out[i * 2..i * 2 + 2].copy_from_slice(&b[i].to_le_bytes());
223                }
224                out
225            }
226            #[inline(always)]
227            fn store_u16_lo(self, b: &mut [u16], v: Self::V) -> () {
228                for i in 0..4 {
229                    b[i] = u16at(&v, i * 2);
230                }
231            }
232            #[inline(always)]
233            fn store_i32_half(self, b: &mut [i32], v: Self::V) -> () {
234                for i in 0..$n / 8 {
235                    b[i] = u32at(&v, i * 4) as i32;
236                }
237            }
238            #[inline(always)]
239            fn add8(self, a: Self::V, b: Self::V) -> Self::V {
240                let mut out = [0u8; $n];
241                for i in (0..$n).step_by(1) {
242                    out[i] = a[i].wrapping_add(b[i]);
243                }
244                out
245            }
246            #[inline(always)]
247            fn sub8(self, a: Self::V, b: Self::V) -> Self::V {
248                let mut out = [0u8; $n];
249                for i in (0..$n).step_by(1) {
250                    out[i] = a[i].wrapping_sub(b[i]);
251                }
252                out
253            }
254            #[inline(always)]
255            fn cmpeq8(self, a: Self::V, b: Self::V) -> Self::V {
256                let mut out = [0u8; $n];
257                for i in (0..$n).step_by(1) {
258                    out[i] = if a[i] == b[i] { u8::MAX } else { 0 };
259                }
260                out
261            }
262            #[inline(always)]
263            fn add16(self, a: Self::V, b: Self::V) -> Self::V {
264                let mut out = [0u8; $n];
265                for i in (0..$n).step_by(2) {
266                    out[i..i + 2]
267                        .copy_from_slice(&(u16at(&a, i).wrapping_add(u16at(&b, i))).to_le_bytes());
268                }
269                out
270            }
271            #[inline(always)]
272            fn sub16(self, a: Self::V, b: Self::V) -> Self::V {
273                let mut out = [0u8; $n];
274                for i in (0..$n).step_by(2) {
275                    out[i..i + 2]
276                        .copy_from_slice(&(u16at(&a, i).wrapping_sub(u16at(&b, i))).to_le_bytes());
277                }
278                out
279            }
280            #[inline(always)]
281            fn cmpeq16(self, a: Self::V, b: Self::V) -> Self::V {
282                let mut out = [0u8; $n];
283                for i in (0..$n).step_by(2) {
284                    out[i..i + 2].copy_from_slice(
285                        &(if u16at(&a, i) == u16at(&b, i) {
286                            u16::MAX
287                        } else {
288                            0
289                        })
290                        .to_le_bytes(),
291                    );
292                }
293                out
294            }
295            #[inline(always)]
296            fn add32(self, a: Self::V, b: Self::V) -> Self::V {
297                let mut out = [0u8; $n];
298                for i in (0..$n).step_by(4) {
299                    out[i..i + 4]
300                        .copy_from_slice(&(u32at(&a, i).wrapping_add(u32at(&b, i))).to_le_bytes());
301                }
302                out
303            }
304            #[inline(always)]
305            fn sub32(self, a: Self::V, b: Self::V) -> Self::V {
306                let mut out = [0u8; $n];
307                for i in (0..$n).step_by(4) {
308                    out[i..i + 4]
309                        .copy_from_slice(&(u32at(&a, i).wrapping_sub(u32at(&b, i))).to_le_bytes());
310                }
311                out
312            }
313            #[inline(always)]
314            fn cmpeq32(self, a: Self::V, b: Self::V) -> Self::V {
315                let mut out = [0u8; $n];
316                for i in (0..$n).step_by(4) {
317                    out[i..i + 4].copy_from_slice(
318                        &(if u32at(&a, i) == u32at(&b, i) {
319                            u32::MAX
320                        } else {
321                            0
322                        })
323                        .to_le_bytes(),
324                    );
325                }
326                out
327            }
328            #[inline(always)]
329            fn add64(self, a: Self::V, b: Self::V) -> Self::V {
330                let mut out = [0u8; $n];
331                for i in (0..$n).step_by(8) {
332                    out[i..i + 8]
333                        .copy_from_slice(&(u64at(&a, i).wrapping_add(u64at(&b, i))).to_le_bytes());
334                }
335                out
336            }
337            #[inline(always)]
338            fn subus8(self, a: Self::V, b: Self::V) -> Self::V {
339                let mut out = [0u8; $n];
340                for i in (0..$n).step_by(1) {
341                    out[i] = a[i].saturating_sub(b[i]);
342                }
343                out
344            }
345            #[inline(always)]
346            fn addus8(self, a: Self::V, b: Self::V) -> Self::V {
347                let mut out = [0u8; $n];
348                for i in (0..$n).step_by(1) {
349                    out[i] = a[i].saturating_add(b[i]);
350                }
351                out
352            }
353            #[inline(always)]
354            fn maxu8(self, a: Self::V, b: Self::V) -> Self::V {
355                let mut out = [0u8; $n];
356                for i in (0..$n).step_by(1) {
357                    out[i] = a[i].max(b[i]);
358                }
359                out
360            }
361            #[inline(always)]
362            fn minu8(self, a: Self::V, b: Self::V) -> Self::V {
363                let mut out = [0u8; $n];
364                for i in (0..$n).step_by(1) {
365                    out[i] = a[i].min(b[i]);
366                }
367                out
368            }
369            #[inline(always)]
370            fn avgu8(self, a: Self::V, b: Self::V) -> Self::V {
371                let mut out = [0u8; $n];
372                for i in (0..$n).step_by(1) {
373                    out[i] = ((a[i] as u16 + b[i] as u16 + 1) >> 1) as u8;
374                }
375                out
376            }
377            #[inline(always)]
378            fn adds8(self, a: Self::V, b: Self::V) -> Self::V {
379                let mut out = [0u8; $n];
380                for i in (0..$n).step_by(1) {
381                    out[i] = (a[i] as i8).saturating_add(b[i] as i8) as u8;
382                }
383                out
384            }
385            #[inline(always)]
386            fn subs8(self, a: Self::V, b: Self::V) -> Self::V {
387                let mut out = [0u8; $n];
388                for i in (0..$n).step_by(1) {
389                    out[i] = (a[i] as i8).saturating_sub(b[i] as i8) as u8;
390                }
391                out
392            }
393            #[inline(always)]
394            fn subus16(self, a: Self::V, b: Self::V) -> Self::V {
395                let mut out = [0u8; $n];
396                for i in (0..$n).step_by(2) {
397                    out[i..i + 2].copy_from_slice(
398                        &(u16at(&a, i).saturating_sub(u16at(&b, i))).to_le_bytes(),
399                    );
400                }
401                out
402            }
403            #[inline(always)]
404            fn mullo16(self, a: Self::V, b: Self::V) -> Self::V {
405                let mut out = [0u8; $n];
406                for i in (0..$n).step_by(2) {
407                    out[i..i + 2]
408                        .copy_from_slice(&(u16at(&a, i).wrapping_mul(u16at(&b, i))).to_le_bytes());
409                }
410                out
411            }
412            #[inline(always)]
413            fn mulhiu16(self, a: Self::V, b: Self::V) -> Self::V {
414                let mut out = [0u8; $n];
415                for i in (0..$n).step_by(2) {
416                    out[i..i + 2].copy_from_slice(
417                        &(((u16at(&a, i) as u32 * u16at(&b, i) as u32) >> 16) as u16).to_le_bytes(),
418                    );
419                }
420                out
421            }
422            #[inline(always)]
423            fn mulhis16(self, a: Self::V, b: Self::V) -> Self::V {
424                let mut out = [0u8; $n];
425                for i in (0..$n).step_by(2) {
426                    out[i..i + 2].copy_from_slice(
427                        &(((u16at(&a, i) as i16 as i32 * u16at(&b, i) as i16 as i32) >> 16) as u16)
428                            .to_le_bytes(),
429                    );
430                }
431                out
432            }
433            #[inline(always)]
434            fn mins16(self, a: Self::V, b: Self::V) -> Self::V {
435                let mut out = [0u8; $n];
436                for i in (0..$n).step_by(2) {
437                    out[i..i + 2].copy_from_slice(
438                        &((u16at(&a, i) as i16).min(u16at(&b, i) as i16) as u16).to_le_bytes(),
439                    );
440                }
441                out
442            }
443            #[inline(always)]
444            fn maxs16(self, a: Self::V, b: Self::V) -> Self::V {
445                let mut out = [0u8; $n];
446                for i in (0..$n).step_by(2) {
447                    out[i..i + 2].copy_from_slice(
448                        &((u16at(&a, i) as i16).max(u16at(&b, i) as i16) as u16).to_le_bytes(),
449                    );
450                }
451                out
452            }
453            #[inline(always)]
454            fn cmpgts16(self, a: Self::V, b: Self::V) -> Self::V {
455                let mut out = [0u8; $n];
456                for i in (0..$n).step_by(2) {
457                    out[i..i + 2].copy_from_slice(
458                        &(if (u16at(&a, i) as i16) > (u16at(&b, i) as i16) {
459                            u16::MAX
460                        } else {
461                            0
462                        })
463                        .to_le_bytes(),
464                    );
465                }
466                out
467            }
468            #[inline(always)]
469            fn srl16<const N: i32>(self, a: Self::V) -> Self::V {
470                let mut out = [0u8; $n];
471                for i in (0..$n).step_by(2) {
472                    out[i..i + 2].copy_from_slice(
473                        &(if (N as u32) < 16 {
474                            u16at(&a, i) >> (N as u32)
475                        } else {
476                            0
477                        })
478                        .to_le_bytes(),
479                    );
480                }
481                out
482            }
483            #[inline(always)]
484            fn sll16<const N: i32>(self, a: Self::V) -> Self::V {
485                let mut out = [0u8; $n];
486                for i in (0..$n).step_by(2) {
487                    out[i..i + 2].copy_from_slice(
488                        &(if (N as u32) < 16 {
489                            u16at(&a, i) << (N as u32)
490                        } else {
491                            0
492                        })
493                        .to_le_bytes(),
494                    );
495                }
496                out
497            }
498            #[inline(always)]
499            fn sra16<const N: i32>(self, a: Self::V) -> Self::V {
500                let mut out = [0u8; $n];
501                for i in (0..$n).step_by(2) {
502                    out[i..i + 2].copy_from_slice(
503                        &(((u16at(&a, i) as i16) >> (N as u32).min(15)) as u16).to_le_bytes(),
504                    );
505                }
506                out
507            }
508            #[inline(always)]
509            fn cmpgts32(self, a: Self::V, b: Self::V) -> Self::V {
510                let mut out = [0u8; $n];
511                for i in (0..$n).step_by(4) {
512                    out[i..i + 4].copy_from_slice(
513                        &(if (u32at(&a, i) as i32) > (u32at(&b, i) as i32) {
514                            u32::MAX
515                        } else {
516                            0
517                        })
518                        .to_le_bytes(),
519                    );
520                }
521                out
522            }
523            #[inline(always)]
524            fn srl32<const N: i32>(self, a: Self::V) -> Self::V {
525                let mut out = [0u8; $n];
526                for i in (0..$n).step_by(4) {
527                    out[i..i + 4].copy_from_slice(
528                        &(if (N as u32) < 32 {
529                            u32at(&a, i) >> (N as u32)
530                        } else {
531                            0
532                        })
533                        .to_le_bytes(),
534                    );
535                }
536                out
537            }
538            #[inline(always)]
539            fn sll32<const N: i32>(self, a: Self::V) -> Self::V {
540                let mut out = [0u8; $n];
541                for i in (0..$n).step_by(4) {
542                    out[i..i + 4].copy_from_slice(
543                        &(if (N as u32) < 32 {
544                            u32at(&a, i) << (N as u32)
545                        } else {
546                            0
547                        })
548                        .to_le_bytes(),
549                    );
550                }
551                out
552            }
553            #[inline(always)]
554            fn sra32<const N: i32>(self, a: Self::V) -> Self::V {
555                let mut out = [0u8; $n];
556                for i in (0..$n).step_by(4) {
557                    out[i..i + 4].copy_from_slice(
558                        &(((u32at(&a, i) as i32) >> (N as u32).min(31)) as u32).to_le_bytes(),
559                    );
560                }
561                out
562            }
563            #[inline(always)]
564            fn srl16x(self, a: Self::V, n: u32) -> Self::V {
565                let mut out = [0u8; $n];
566                for i in (0..$n).step_by(2) {
567                    out[i..i + 2].copy_from_slice(
568                        &(if n < 16 { u16at(&a, i) >> n } else { 0 }).to_le_bytes(),
569                    );
570                }
571                out
572            }
573            #[inline(always)]
574            fn movemask8(self, a: Self::V) -> u32 {
575                let mut out = 0;
576                for i in (0..$n).step_by(1) {
577                    out |= ((a[i] >> 7) as u32) << (i / 1);
578                }
579                out
580            }
581            #[inline(always)]
582            fn movemask32(self, a: Self::V) -> u32 {
583                let mut out = 0;
584                for i in (0..$n).step_by(4) {
585                    out |= ((u32at(&a, i) >> 31) as u32) << (i / 4);
586                }
587                out
588            }
589            #[inline(always)]
590            fn unpacklo8(self, a: Self::V, b: Self::V) -> Self::V {
591                let mut out = [0; $n];
592                for lane in (0..$n).step_by(16) {
593                    for j in (0..8).step_by(1) {
594                        let dst = lane + j * 2;
595                        let src = lane + 0 + j;
596                        out[dst..dst + 1].copy_from_slice(&a[src..src + 1]);
597                        out[dst + 1..dst + 2].copy_from_slice(&b[src..src + 1]);
598                    }
599                }
600                out
601            }
602            #[inline(always)]
603            fn unpackhi8(self, a: Self::V, b: Self::V) -> Self::V {
604                let mut out = [0; $n];
605                for lane in (0..$n).step_by(16) {
606                    for j in (0..8).step_by(1) {
607                        let dst = lane + j * 2;
608                        let src = lane + 8 + j;
609                        out[dst..dst + 1].copy_from_slice(&a[src..src + 1]);
610                        out[dst + 1..dst + 2].copy_from_slice(&b[src..src + 1]);
611                    }
612                }
613                out
614            }
615            #[inline(always)]
616            fn unpacklo16(self, a: Self::V, b: Self::V) -> Self::V {
617                let mut out = [0; $n];
618                for lane in (0..$n).step_by(16) {
619                    for j in (0..8).step_by(2) {
620                        let dst = lane + j * 2;
621                        let src = lane + 0 + j;
622                        out[dst..dst + 2].copy_from_slice(&a[src..src + 2]);
623                        out[dst + 2..dst + 4].copy_from_slice(&b[src..src + 2]);
624                    }
625                }
626                out
627            }
628            #[inline(always)]
629            fn unpackhi16(self, a: Self::V, b: Self::V) -> Self::V {
630                let mut out = [0; $n];
631                for lane in (0..$n).step_by(16) {
632                    for j in (0..8).step_by(2) {
633                        let dst = lane + j * 2;
634                        let src = lane + 8 + j;
635                        out[dst..dst + 2].copy_from_slice(&a[src..src + 2]);
636                        out[dst + 2..dst + 4].copy_from_slice(&b[src..src + 2]);
637                    }
638                }
639                out
640            }
641            #[inline(always)]
642            fn packus16(self, a: Self::V, b: Self::V) -> Self::V {
643                let mut out = [0; $n];
644                for lane in (0..$n).step_by(16) {
645                    for half in 0..2 {
646                        let src = if half == 0 { &a } else { &b };
647                        for j in 0..8 {
648                            let i = lane + half * 8 + j * 1;
649                            out[i] = (u16at(src, lane + j * 2) as i16).clamp(0, 255) as u8;
650                        }
651                    }
652                }
653                out
654            }
655            #[inline(always)]
656            fn packs16(self, a: Self::V, b: Self::V) -> Self::V {
657                let mut out = [0; $n];
658                for lane in (0..$n).step_by(16) {
659                    for half in 0..2 {
660                        let src = if half == 0 { &a } else { &b };
661                        for j in 0..8 {
662                            let i = lane + half * 8 + j * 1;
663                            out[i] = (u16at(src, lane + j * 2) as i16).clamp(-128, 127) as u8;
664                        }
665                    }
666                }
667                out
668            }
669            #[inline(always)]
670            fn packs32(self, a: Self::V, b: Self::V) -> Self::V {
671                let mut out = [0; $n];
672                for lane in (0..$n).step_by(16) {
673                    for half in 0..2 {
674                        let src = if half == 0 { &a } else { &b };
675                        for j in 0..4 {
676                            let i = lane + half * 8 + j * 2;
677                            out[i..i + 2].copy_from_slice(
678                                &((u32at(src, lane + j * 4) as i32).clamp(-32768, 32767) as u16)
679                                    .to_le_bytes(),
680                            );
681                        }
682                    }
683                }
684                out
685            }
686            #[inline(always)]
687            fn packssdw(self, a: Self::V, b: Self::V) -> Self::V {
688                self.packs32(a, b)
689            }
690            #[inline(always)]
691            fn packuswb_px(self, v: Self::V) -> Self::V {
692                let mut out = [0; $n];
693                for i in 0..$n / 2 {
694                    out[i] = (u16at(&v, i * 2) as i16).clamp(0, 255) as u8;
695                }
696                for i in 0..$n / 2 {
697                    out[$n / 2 + i] = out[i % 8];
698                }
699                out
700            }
701            #[inline(always)]
702            fn madd16(self, a: Self::V, b: Self::V) -> Self::V {
703                let mut out = [0u8; $n];
704                for i in (0..$n).step_by(4) {
705                    out[i..i + 4].copy_from_slice(
706                        &((u16at(&a, i) as i16 as i32)
707                            .wrapping_mul(u16at(&b, i) as i16 as i32)
708                            .wrapping_add((u16at(&a, i + 2) as i16 as i32).wrapping_mul(u16at(
709                                &b,
710                                i + 2,
711                            )
712                                as i16
713                                as i32)) as u32)
714                            .to_le_bytes(),
715                    );
716                }
717                out
718            }
719            #[inline(always)]
720            fn shuf16<const IMM: i32>(self, a: Self::V) -> Self::V {
721                let mut out = [0; $n];
722                for half in (0..$n).step_by(8) {
723                    for j in 0..4 {
724                        let src = half + ((IMM as usize >> (j * 2)) & 3) * 2;
725                        out[half + j * 2..half + j * 2 + 2].copy_from_slice(&a[src..src + 2]);
726                    }
727                }
728                out
729            }
730            #[inline(always)]
731            fn shuf32<const IMM: i32>(self, a: Self::V) -> Self::V {
732                let mut out = [0; $n];
733                for lane in (0..$n).step_by(16) {
734                    for j in 0..4 {
735                        let src = lane + ((IMM as usize >> (j * 2)) & 3) * 4;
736                        out[lane + j * 4..lane + j * 4 + 4].copy_from_slice(&a[src..src + 4]);
737                    }
738                }
739                out
740            }
741            #[inline(always)]
742            fn shufps<const IMM: i32>(self, a: Self::V, b: Self::V) -> Self::V {
743                let mut out = [0; $n];
744                for lane in (0..$n).step_by(16) {
745                    for j in 0..4 {
746                        let src = lane + ((IMM as usize >> (j * 2)) & 3) * 4;
747                        let from = if j < 2 { &a } else { &b };
748                        out[lane + j * 4..lane + j * 4 + 4].copy_from_slice(&from[src..src + 4]);
749                    }
750                }
751                out
752            }
753            #[inline(always)]
754            fn bsll<const N: i32>(self, a: Self::V) -> Self::V {
755                let mut out = [0; $n];
756                let n = N as usize;
757                if n < 16 {
758                    for lane in (0..$n).step_by(16) {
759                        out[lane + n..lane + 16].copy_from_slice(&a[lane..lane + 16 - n]);
760                    }
761                }
762                out
763            }
764            #[inline(always)]
765            fn bsrl<const N: i32>(self, a: Self::V) -> Self::V {
766                let mut out = [0; $n];
767                let n = N as usize;
768                if n < 16 {
769                    for lane in (0..$n).step_by(16) {
770                        out[lane..lane + 16 - n].copy_from_slice(&a[lane + n..lane + 16]);
771                    }
772                }
773                out
774            }
775            #[inline(always)]
776            fn and(self, a: Self::V, b: Self::V) -> Self::V {
777                let mut out = [0u8; $n];
778                for i in (0..$n).step_by(1) {
779                    out[i] = a[i] & b[i];
780                }
781                out
782            }
783            #[inline(always)]
784            fn andnot(self, a: Self::V, b: Self::V) -> Self::V {
785                let mut out = [0u8; $n];
786                for i in (0..$n).step_by(1) {
787                    out[i] = !a[i] & b[i];
788                }
789                out
790            }
791            #[inline(always)]
792            fn or(self, a: Self::V, b: Self::V) -> Self::V {
793                let mut out = [0u8; $n];
794                for i in (0..$n).step_by(1) {
795                    out[i] = a[i] | b[i];
796                }
797                out
798            }
799            #[inline(always)]
800            fn xor(self, a: Self::V, b: Self::V) -> Self::V {
801                let mut out = [0u8; $n];
802                for i in (0..$n).step_by(1) {
803                    out[i] = a[i] ^ b[i];
804                }
805                out
806            }
807            #[inline(always)]
808            fn addf(self, a: Self::V, b: Self::V) -> Self::V {
809                let mut out = [0u8; $n];
810                for i in (0..$n).step_by(4) {
811                    out[i..i + 4].copy_from_slice(
812                        &(portable_fx::arithmetic32(
813                            f32::from_bits(u32at(&a, i)),
814                            f32::from_bits(u32at(&b, i)),
815                            f32::from_bits(u32at(&a, i)) + f32::from_bits(u32at(&b, i)),
816                        )
817                        .to_bits())
818                        .to_le_bytes(),
819                    );
820                }
821                out
822            }
823            #[inline(always)]
824            fn subf(self, a: Self::V, b: Self::V) -> Self::V {
825                let mut out = [0u8; $n];
826                for i in (0..$n).step_by(4) {
827                    out[i..i + 4].copy_from_slice(
828                        &(portable_fx::arithmetic32(
829                            f32::from_bits(u32at(&a, i)),
830                            f32::from_bits(u32at(&b, i)),
831                            f32::from_bits(u32at(&a, i)) - f32::from_bits(u32at(&b, i)),
832                        )
833                        .to_bits())
834                        .to_le_bytes(),
835                    );
836                }
837                out
838            }
839            #[inline(always)]
840            fn mulf(self, a: Self::V, b: Self::V) -> Self::V {
841                let mut out = [0u8; $n];
842                for i in (0..$n).step_by(4) {
843                    out[i..i + 4].copy_from_slice(
844                        &(portable_fx::arithmetic32(
845                            f32::from_bits(u32at(&a, i)),
846                            f32::from_bits(u32at(&b, i)),
847                            f32::from_bits(u32at(&a, i)) * f32::from_bits(u32at(&b, i)),
848                        )
849                        .to_bits())
850                        .to_le_bytes(),
851                    );
852                }
853                out
854            }
855            #[inline(always)]
856            fn divf(self, a: Self::V, b: Self::V) -> Self::V {
857                let mut out = [0u8; $n];
858                for i in (0..$n).step_by(4) {
859                    out[i..i + 4].copy_from_slice(
860                        &(portable_fx::arithmetic32(
861                            f32::from_bits(u32at(&a, i)),
862                            f32::from_bits(u32at(&b, i)),
863                            f32::from_bits(u32at(&a, i)) / f32::from_bits(u32at(&b, i)),
864                        )
865                        .to_bits())
866                        .to_le_bytes(),
867                    );
868                }
869                out
870            }
871            #[inline(always)]
872            fn maxf(self, a: Self::V, b: Self::V) -> Self::V {
873                let mut out = [0u8; $n];
874                for i in (0..$n).step_by(4) {
875                    out[i..i + 4].copy_from_slice(
876                        &(if f32::from_bits(u32at(&a, i)) > f32::from_bits(u32at(&b, i)) {
877                            u32at(&a, i)
878                        } else {
879                            u32at(&b, i)
880                        })
881                        .to_le_bytes(),
882                    );
883                }
884                out
885            }
886            #[inline(always)]
887            fn minf(self, a: Self::V, b: Self::V) -> Self::V {
888                let mut out = [0u8; $n];
889                for i in (0..$n).step_by(4) {
890                    out[i..i + 4].copy_from_slice(
891                        &(if f32::from_bits(u32at(&a, i)) < f32::from_bits(u32at(&b, i)) {
892                            u32at(&a, i)
893                        } else {
894                            u32at(&b, i)
895                        })
896                        .to_le_bytes(),
897                    );
898                }
899                out
900            }
901            #[inline(always)]
902            fn cmpltf(self, a: Self::V, b: Self::V) -> Self::V {
903                let mut out = [0u8; $n];
904                for i in (0..$n).step_by(4) {
905                    out[i..i + 4].copy_from_slice(
906                        &(if f32::from_bits(u32at(&a, i)) < f32::from_bits(u32at(&b, i)) {
907                            u32::MAX
908                        } else {
909                            0
910                        })
911                        .to_le_bytes(),
912                    );
913                }
914                out
915            }
916            #[inline(always)]
917            fn sqrtf(self, a: Self::V) -> Self::V {
918                let mut out = [0u8; $n];
919                for i in (0..$n).step_by(4) {
920                    out[i..i + 4].copy_from_slice(
921                        &(rt::fx::sqrt(f32::from_bits(u32at(&a, i))).to_bits()).to_le_bytes(),
922                    );
923                }
924                out
925            }
926            #[inline(always)]
927            fn addd(self, a: Self::V, b: Self::V) -> Self::V {
928                let mut out = [0u8; $n];
929                for i in (0..$n).step_by(8) {
930                    out[i..i + 8].copy_from_slice(
931                        &(portable_fx::arithmetic64(
932                            f64::from_bits(u64at(&a, i)),
933                            f64::from_bits(u64at(&b, i)),
934                            f64::from_bits(u64at(&a, i)) + f64::from_bits(u64at(&b, i)),
935                        )
936                        .to_bits())
937                        .to_le_bytes(),
938                    );
939                }
940                out
941            }
942            #[inline(always)]
943            fn subd(self, a: Self::V, b: Self::V) -> Self::V {
944                let mut out = [0u8; $n];
945                for i in (0..$n).step_by(8) {
946                    out[i..i + 8].copy_from_slice(
947                        &(portable_fx::arithmetic64(
948                            f64::from_bits(u64at(&a, i)),
949                            f64::from_bits(u64at(&b, i)),
950                            f64::from_bits(u64at(&a, i)) - f64::from_bits(u64at(&b, i)),
951                        )
952                        .to_bits())
953                        .to_le_bytes(),
954                    );
955                }
956                out
957            }
958            #[inline(always)]
959            fn muld(self, a: Self::V, b: Self::V) -> Self::V {
960                let mut out = [0u8; $n];
961                for i in (0..$n).step_by(8) {
962                    out[i..i + 8].copy_from_slice(
963                        &(portable_fx::arithmetic64(
964                            f64::from_bits(u64at(&a, i)),
965                            f64::from_bits(u64at(&b, i)),
966                            f64::from_bits(u64at(&a, i)) * f64::from_bits(u64at(&b, i)),
967                        )
968                        .to_bits())
969                        .to_le_bytes(),
970                    );
971                }
972                out
973            }
974            #[inline(always)]
975            fn divd(self, a: Self::V, b: Self::V) -> Self::V {
976                let mut out = [0u8; $n];
977                for i in (0..$n).step_by(8) {
978                    out[i..i + 8].copy_from_slice(
979                        &(portable_fx::arithmetic64(
980                            f64::from_bits(u64at(&a, i)),
981                            f64::from_bits(u64at(&b, i)),
982                            f64::from_bits(u64at(&a, i)) / f64::from_bits(u64at(&b, i)),
983                        )
984                        .to_bits())
985                        .to_le_bytes(),
986                    );
987                }
988                out
989            }
990            #[inline(always)]
991            fn maxd(self, a: Self::V, b: Self::V) -> Self::V {
992                let mut out = [0u8; $n];
993                for i in (0..$n).step_by(8) {
994                    out[i..i + 8].copy_from_slice(
995                        &(if f64::from_bits(u64at(&a, i)) > f64::from_bits(u64at(&b, i)) {
996                            u64at(&a, i)
997                        } else {
998                            u64at(&b, i)
999                        })
1000                        .to_le_bytes(),
1001                    );
1002                }
1003                out
1004            }
1005            #[inline(always)]
1006            fn mind(self, a: Self::V, b: Self::V) -> Self::V {
1007                let mut out = [0u8; $n];
1008                for i in (0..$n).step_by(8) {
1009                    out[i..i + 8].copy_from_slice(
1010                        &(if f64::from_bits(u64at(&a, i)) < f64::from_bits(u64at(&b, i)) {
1011                            u64at(&a, i)
1012                        } else {
1013                            u64at(&b, i)
1014                        })
1015                        .to_le_bytes(),
1016                    );
1017                }
1018                out
1019            }
1020            #[inline(always)]
1021            fn cmpltd(self, a: Self::V, b: Self::V) -> Self::V {
1022                let mut out = [0u8; $n];
1023                for i in (0..$n).step_by(8) {
1024                    out[i..i + 8].copy_from_slice(
1025                        &(if f64::from_bits(u64at(&a, i)) < f64::from_bits(u64at(&b, i)) {
1026                            u64::MAX
1027                        } else {
1028                            0
1029                        })
1030                        .to_le_bytes(),
1031                    );
1032                }
1033                out
1034            }
1035            #[inline(always)]
1036            fn sqrtd(self, a: Self::V) -> Self::V {
1037                let mut out = [0u8; $n];
1038                for i in (0..$n).step_by(8) {
1039                    out[i..i + 8].copy_from_slice(
1040                        &(rt::fx::sqrtd(f64::from_bits(u64at(&a, i))).to_bits()).to_le_bytes(),
1041                    );
1042                }
1043                out
1044            }
1045            #[inline(always)]
1046            fn cvt_i32_f32(self, a: Self::V) -> Self::V {
1047                let mut out = [0u8; $n];
1048                for i in (0..$n).step_by(4) {
1049                    out[i..i + 4]
1050                        .copy_from_slice(&((u32at(&a, i) as i32 as f32).to_bits()).to_le_bytes());
1051                }
1052                out
1053            }
1054            #[inline(always)]
1055            fn cvtt_f32_i32(self, a: Self::V) -> Self::V {
1056                let mut out = [0u8; $n];
1057                for i in (0..$n).step_by(4) {
1058                    out[i..i + 4].copy_from_slice(
1059                        &(rt::fx::cvtt(f32::from_bits(u32at(&a, i))) as u32).to_le_bytes(),
1060                    );
1061                }
1062                out
1063            }
1064            #[inline(always)]
1065            fn cvtr_f32_i32(self, a: Self::V) -> Self::V {
1066                let mut out = [0u8; $n];
1067                for i in (0..$n).step_by(4) {
1068                    out[i..i + 4].copy_from_slice(
1069                        &(rt::fx::cvt(f32::from_bits(u32at(&a, i))) as u32).to_le_bytes(),
1070                    );
1071                }
1072                out
1073            }
1074            #[inline(always)]
1075            fn cvt_i32_f64(self, a: Self::V) -> Self::V {
1076                let mut out = [0u8; $n];
1077                for i in (0..$n).step_by(8) {
1078                    out[i..i + 8].copy_from_slice(
1079                        &((u32at(&a, i / 2) as i32 as f64).to_bits()).to_le_bytes(),
1080                    );
1081                }
1082                out
1083            }
1084            #[inline(always)]
1085            fn cvtt_f64_i32(self, a: Self::V) -> Self::V {
1086                let mut out = [0; $n];
1087                for i in 0..$n / 8 {
1088                    out[i * 4..i * 4 + 4].copy_from_slice(
1089                        &rt::fx::cvttd(f64::from_bits(u64at(&a, i * 8))).to_le_bytes(),
1090                    );
1091                }
1092                out
1093            }
1094            #[inline(always)]
1095            fn store_lanes2(self, b: &mut [u8], lo: Self::V, hi: Self::V) -> () {
1096                for lane in 0..$n / 16 {
1097                    b[lane * 32..lane * 32 + 16].copy_from_slice(&lo[lane * 16..lane * 16 + 16]);
1098                    b[lane * 32 + 16..lane * 32 + 32]
1099                        .copy_from_slice(&hi[lane * 16..lane * 16 + 16]);
1100                }
1101            }
1102            #[inline(always)]
1103            fn load_lanes2(self, b: &[u8]) -> (Self::V, Self::V) {
1104                let mut lo = [0; $n];
1105                let mut hi = [0; $n];
1106                for lane in 0..$n / 16 {
1107                    lo[lane * 16..lane * 16 + 16].copy_from_slice(&b[lane * 32..lane * 32 + 16]);
1108                    hi[lane * 16..lane * 16 + 16]
1109                        .copy_from_slice(&b[lane * 32 + 16..lane * 32 + 32]);
1110                }
1111                (lo, hi)
1112            }
1113            #[inline(always)]
1114            fn load_i16_lanes(self, a: &[i16], b: &[i16]) -> Self::V {
1115                let mut out = [0; $n];
1116                for lane in 0..$n / 16 {
1117                    let from = if lane == 0 { a } else { b };
1118                    for i in 0..8 {
1119                        let dst = lane * 16 + i * 2;
1120                        out[dst..dst + 2].copy_from_slice(&from[i].to_le_bytes());
1121                    }
1122                }
1123                out
1124            }
1125            #[inline(always)]
1126            fn store_raw_lanes(self, a: &mut [u8], b: &mut [u8], v: Self::V) -> () {
1127                a[..8].copy_from_slice(&v[..8]);
1128                if $n > 16 {
1129                    b[..8].copy_from_slice(&v[16..24]);
1130                }
1131            }
1132            #[inline(always)]
1133            fn load_f32_lanes(self, a: &[f32], b: &[f32]) -> Self::V {
1134                let mut out = [0; $n];
1135                for lane in 0..$n / 16 {
1136                    let from = if lane == 0 { a } else { b };
1137                    for i in 0..4 {
1138                        let dst = lane * 16 + i * 4;
1139                        out[dst..dst + 4].copy_from_slice(&from[i].to_bits().to_le_bytes());
1140                    }
1141                }
1142                out
1143            }
1144            #[inline(always)]
1145            fn store_f32_lanes(self, a: &mut [f32], b: &mut [f32], v: Self::V) -> () {
1146                for i in 0..4 {
1147                    a[i] = f32::from_bits(u32at(&v, i * 4));
1148                }
1149                if $n > 16 {
1150                    for i in 0..4 {
1151                        b[i] = f32::from_bits(u32at(&v, 16 + i * 4));
1152                    }
1153                }
1154            }
1155            #[inline(always)]
1156            fn sadu8(self, a: Self::V, b: Self::V) -> Self::V {
1157                let mut out = [0; $n];
1158                for lane in (0..$n).step_by(8) {
1159                    let mut sum = 0u64;
1160                    for i in 0..8 {
1161                        sum += a[lane + i].abs_diff(b[lane + i]) as u64;
1162                    }
1163                    out[lane..lane + 8].copy_from_slice(&sum.to_le_bytes());
1164                }
1165                out
1166            }
1167            #[inline(always)]
1168            fn ptest_raw(self, a: Self::V, b: Self::V) -> u32 {
1169                let mut z = 0;
1170                let mut c = 0;
1171                for i in 0..$n / 2 {
1172                    z |= a[i] & b[i];
1173                    c |= !a[i] & b[i];
1174                }
1175                if z == 0 {
1176                    1
1177                } else if c == 0 {
1178                    2
1179                } else {
1180                    0
1181                }
1182            }
1183            #[inline(always)]
1184            fn ptestz_raw(self, a: Self::V, b: Self::V) -> bool {
1185                self.ptest_raw(a, b) == 1
1186            }
1187        }
1188    };
1189}
1190backend!(Scalar, 16);
1191backend!(Scalar256, 32);