1use crate::Simd;
6#[cfg(target_arch = "wasm32")]
7use rt::fx as portable_fx;
8#[cfg(not(target_arch = "wasm32"))]
9use rt::portable::fx as portable_fx;
10
11#[derive(Clone, Copy)]
13pub struct Scalar;
14#[derive(Clone, Copy)]
16pub struct Scalar256;
17impl Scalar256 {
18 pub fn new() -> Option<Self> {
20 None
21 }
22}
23
24#[inline(always)]
25fn u16at(a: &[u8], i: usize) -> u16 {
26 u16::from_le_bytes([a[i], a[i + 1]])
27}
28#[inline(always)]
29fn u32at(a: &[u8], i: usize) -> u32 {
30 u32::from_le_bytes([a[i], a[i + 1], a[i + 2], a[i + 3]])
31}
32#[inline(always)]
33fn u64at(a: &[u8], i: usize) -> u64 {
34 u64::from_le_bytes([
35 a[i],
36 a[i + 1],
37 a[i + 2],
38 a[i + 3],
39 a[i + 4],
40 a[i + 5],
41 a[i + 6],
42 a[i + 7],
43 ])
44}
45
46macro_rules! backend {
47 ($name:ident, $n:expr) => {
48 impl Simd for $name {
49 type V = [u8; $n];
50 const BYTES: usize = $n;
51 const PX: usize = $n / 8;
52 #[inline(always)]
53 fn zero(self) -> Self::V {
54 [0; $n]
55 }
56 #[inline(always)]
57 fn splat8(self, v: u8) -> Self::V {
58 let mut out = [0u8; $n];
59 for i in (0..$n).step_by(1) {
60 out[i] = v;
61 }
62 out
63 }
64 #[inline(always)]
65 fn splat16(self, v: u16) -> Self::V {
66 let mut out = [0u8; $n];
67 for i in (0..$n).step_by(2) {
68 out[i..i + 2].copy_from_slice(&(v).to_le_bytes());
69 }
70 out
71 }
72 #[inline(always)]
73 fn splat32(self, v: u32) -> Self::V {
74 let mut out = [0u8; $n];
75 for i in (0..$n).step_by(4) {
76 out[i..i + 4].copy_from_slice(&(v).to_le_bytes());
77 }
78 out
79 }
80 #[inline(always)]
81 fn splat64(self, v: u64) -> Self::V {
82 let mut out = [0u8; $n];
83 for i in (0..$n).step_by(8) {
84 out[i..i + 8].copy_from_slice(&(v).to_le_bytes());
85 }
86 out
87 }
88 #[inline(always)]
89 fn splatf(self, v: f32) -> Self::V {
90 self.splat32(v.to_bits())
91 }
92 #[inline(always)]
93 fn splatd(self, v: f64) -> Self::V {
94 self.splat64(v.to_bits())
95 }
96 #[inline(always)]
97 fn load(self, b: &[u8]) -> Self::V {
98 let mut out = [0; $n];
99 out.copy_from_slice(&b[..$n]);
100 out
101 }
102 #[inline(always)]
103 fn store(self, b: &mut [u8], v: Self::V) -> () {
104 b[..$n].copy_from_slice(&v);
105 }
106 #[inline(always)]
107 fn load_raw(self, b: &[u8]) -> Self::V {
108 let mut out = [0; $n];
109 out[..$n / 2].copy_from_slice(&b[..$n / 2]);
110 out
111 }
112 #[inline(always)]
113 fn store_raw(self, b: &mut [u8], v: Self::V) -> () {
114 b[..$n / 2].copy_from_slice(&v[..$n / 2]);
115 }
116 #[inline(always)]
117 fn widen_px(self, v: Self::V) -> Self::V {
118 let mut out = [0; $n];
119 for i in 0..$n / 2 {
120 out[i * 2] = v[i];
121 }
122 out
123 }
124 #[inline(always)]
125 fn load_px(self, b: &[u8]) -> Self::V {
126 self.widen_px(self.load_raw(b))
127 }
128 #[inline(always)]
129 fn store_px(self, b: &mut [u8], v: Self::V) -> () {
130 for i in 0..$n / 2 {
131 b[i] = (u16at(&v, i * 2) as i16).clamp(0, 255) as u8;
132 }
133 }
134 #[inline(always)]
135 fn bcast_px(self, b: &[u8]) -> Self::V {
136 let mut out = [0; $n];
137 for i in 0..$n / 8 {
138 for j in 0..4 {
139 out[i * 8 + j * 2] = b[i];
140 }
141 }
142 out
143 }
144 #[inline(always)]
145 fn load_lane32(self, b: &[u8]) -> Self::V {
146 let mut out = [0; $n];
147 for lane in 0..$n / 16 {
148 out[lane * 16..lane * 16 + 4].copy_from_slice(&b[lane * 4..lane * 4 + 4]);
149 }
150 out
151 }
152 #[inline(always)]
153 fn from_i32(self, v: i32) -> Self::V {
154 let mut out = [0; $n];
155 out[..4].copy_from_slice(&v.to_le_bytes());
156 out
157 }
158 #[inline(always)]
159 fn to_i32(self, v: Self::V) -> i32 {
160 u32at(&v, 0) as i32
161 }
162 #[inline(always)]
163 fn load_i16(self, b: &[i16]) -> Self::V {
164 let mut out = [0; $n];
165 for i in 0..$n / 2 {
166 out[i * 2..(i + 1) * 2].copy_from_slice(&(b[i]).to_le_bytes());
167 }
168 out
169 }
170 #[inline(always)]
171 fn store_i16(self, b: &mut [i16], v: Self::V) -> () {
172 for i in 0..$n / 2 {
173 b[i] = u16at(&v, i * 2) as i16;
174 }
175 }
176 #[inline(always)]
177 fn load_u16(self, b: &[u16]) -> Self::V {
178 let mut out = [0; $n];
179 for i in 0..$n / 2 {
180 out[i * 2..(i + 1) * 2].copy_from_slice(&(b[i]).to_le_bytes());
181 }
182 out
183 }
184 #[inline(always)]
185 fn store_u16(self, b: &mut [u16], v: Self::V) -> () {
186 for i in 0..$n / 2 {
187 b[i] = u16at(&v, i * 2) as u16;
188 }
189 }
190 #[inline(always)]
191 fn load_i32(self, b: &[i32]) -> Self::V {
192 let mut out = [0; $n];
193 for i in 0..$n / 4 {
194 out[i * 4..(i + 1) * 4].copy_from_slice(&(b[i]).to_le_bytes());
195 }
196 out
197 }
198 #[inline(always)]
199 fn store_i32(self, b: &mut [i32], v: Self::V) -> () {
200 for i in 0..$n / 4 {
201 b[i] = u32at(&v, i * 4) as i32;
202 }
203 }
204 #[inline(always)]
205 fn load_f32(self, b: &[f32]) -> Self::V {
206 let mut out = [0; $n];
207 for i in 0..$n / 4 {
208 out[i * 4..(i + 1) * 4].copy_from_slice(&(b[i].to_bits()).to_le_bytes());
209 }
210 out
211 }
212 #[inline(always)]
213 fn store_f32(self, b: &mut [f32], v: Self::V) -> () {
214 for i in 0..$n / 4 {
215 b[i] = f32::from_bits(u32at(&v, i * 4));
216 }
217 }
218 #[inline(always)]
219 fn load_u16_lo(self, b: &[u16]) -> Self::V {
220 let mut out = [0; $n];
221 for i in 0..4 {
222 out[i * 2..i * 2 + 2].copy_from_slice(&b[i].to_le_bytes());
223 }
224 out
225 }
226 #[inline(always)]
227 fn store_u16_lo(self, b: &mut [u16], v: Self::V) -> () {
228 for i in 0..4 {
229 b[i] = u16at(&v, i * 2);
230 }
231 }
232 #[inline(always)]
233 fn store_i32_half(self, b: &mut [i32], v: Self::V) -> () {
234 for i in 0..$n / 8 {
235 b[i] = u32at(&v, i * 4) as i32;
236 }
237 }
238 #[inline(always)]
239 fn add8(self, a: Self::V, b: Self::V) -> Self::V {
240 let mut out = [0u8; $n];
241 for i in (0..$n).step_by(1) {
242 out[i] = a[i].wrapping_add(b[i]);
243 }
244 out
245 }
246 #[inline(always)]
247 fn sub8(self, a: Self::V, b: Self::V) -> Self::V {
248 let mut out = [0u8; $n];
249 for i in (0..$n).step_by(1) {
250 out[i] = a[i].wrapping_sub(b[i]);
251 }
252 out
253 }
254 #[inline(always)]
255 fn cmpeq8(self, a: Self::V, b: Self::V) -> Self::V {
256 let mut out = [0u8; $n];
257 for i in (0..$n).step_by(1) {
258 out[i] = if a[i] == b[i] { u8::MAX } else { 0 };
259 }
260 out
261 }
262 #[inline(always)]
263 fn add16(self, a: Self::V, b: Self::V) -> Self::V {
264 let mut out = [0u8; $n];
265 for i in (0..$n).step_by(2) {
266 out[i..i + 2]
267 .copy_from_slice(&(u16at(&a, i).wrapping_add(u16at(&b, i))).to_le_bytes());
268 }
269 out
270 }
271 #[inline(always)]
272 fn sub16(self, a: Self::V, b: Self::V) -> Self::V {
273 let mut out = [0u8; $n];
274 for i in (0..$n).step_by(2) {
275 out[i..i + 2]
276 .copy_from_slice(&(u16at(&a, i).wrapping_sub(u16at(&b, i))).to_le_bytes());
277 }
278 out
279 }
280 #[inline(always)]
281 fn cmpeq16(self, a: Self::V, b: Self::V) -> Self::V {
282 let mut out = [0u8; $n];
283 for i in (0..$n).step_by(2) {
284 out[i..i + 2].copy_from_slice(
285 &(if u16at(&a, i) == u16at(&b, i) {
286 u16::MAX
287 } else {
288 0
289 })
290 .to_le_bytes(),
291 );
292 }
293 out
294 }
295 #[inline(always)]
296 fn add32(self, a: Self::V, b: Self::V) -> Self::V {
297 let mut out = [0u8; $n];
298 for i in (0..$n).step_by(4) {
299 out[i..i + 4]
300 .copy_from_slice(&(u32at(&a, i).wrapping_add(u32at(&b, i))).to_le_bytes());
301 }
302 out
303 }
304 #[inline(always)]
305 fn sub32(self, a: Self::V, b: Self::V) -> Self::V {
306 let mut out = [0u8; $n];
307 for i in (0..$n).step_by(4) {
308 out[i..i + 4]
309 .copy_from_slice(&(u32at(&a, i).wrapping_sub(u32at(&b, i))).to_le_bytes());
310 }
311 out
312 }
313 #[inline(always)]
314 fn cmpeq32(self, a: Self::V, b: Self::V) -> Self::V {
315 let mut out = [0u8; $n];
316 for i in (0..$n).step_by(4) {
317 out[i..i + 4].copy_from_slice(
318 &(if u32at(&a, i) == u32at(&b, i) {
319 u32::MAX
320 } else {
321 0
322 })
323 .to_le_bytes(),
324 );
325 }
326 out
327 }
328 #[inline(always)]
329 fn add64(self, a: Self::V, b: Self::V) -> Self::V {
330 let mut out = [0u8; $n];
331 for i in (0..$n).step_by(8) {
332 out[i..i + 8]
333 .copy_from_slice(&(u64at(&a, i).wrapping_add(u64at(&b, i))).to_le_bytes());
334 }
335 out
336 }
337 #[inline(always)]
338 fn subus8(self, a: Self::V, b: Self::V) -> Self::V {
339 let mut out = [0u8; $n];
340 for i in (0..$n).step_by(1) {
341 out[i] = a[i].saturating_sub(b[i]);
342 }
343 out
344 }
345 #[inline(always)]
346 fn addus8(self, a: Self::V, b: Self::V) -> Self::V {
347 let mut out = [0u8; $n];
348 for i in (0..$n).step_by(1) {
349 out[i] = a[i].saturating_add(b[i]);
350 }
351 out
352 }
353 #[inline(always)]
354 fn maxu8(self, a: Self::V, b: Self::V) -> Self::V {
355 let mut out = [0u8; $n];
356 for i in (0..$n).step_by(1) {
357 out[i] = a[i].max(b[i]);
358 }
359 out
360 }
361 #[inline(always)]
362 fn minu8(self, a: Self::V, b: Self::V) -> Self::V {
363 let mut out = [0u8; $n];
364 for i in (0..$n).step_by(1) {
365 out[i] = a[i].min(b[i]);
366 }
367 out
368 }
369 #[inline(always)]
370 fn avgu8(self, a: Self::V, b: Self::V) -> Self::V {
371 let mut out = [0u8; $n];
372 for i in (0..$n).step_by(1) {
373 out[i] = ((a[i] as u16 + b[i] as u16 + 1) >> 1) as u8;
374 }
375 out
376 }
377 #[inline(always)]
378 fn adds8(self, a: Self::V, b: Self::V) -> Self::V {
379 let mut out = [0u8; $n];
380 for i in (0..$n).step_by(1) {
381 out[i] = (a[i] as i8).saturating_add(b[i] as i8) as u8;
382 }
383 out
384 }
385 #[inline(always)]
386 fn subs8(self, a: Self::V, b: Self::V) -> Self::V {
387 let mut out = [0u8; $n];
388 for i in (0..$n).step_by(1) {
389 out[i] = (a[i] as i8).saturating_sub(b[i] as i8) as u8;
390 }
391 out
392 }
393 #[inline(always)]
394 fn subus16(self, a: Self::V, b: Self::V) -> Self::V {
395 let mut out = [0u8; $n];
396 for i in (0..$n).step_by(2) {
397 out[i..i + 2].copy_from_slice(
398 &(u16at(&a, i).saturating_sub(u16at(&b, i))).to_le_bytes(),
399 );
400 }
401 out
402 }
403 #[inline(always)]
404 fn mullo16(self, a: Self::V, b: Self::V) -> Self::V {
405 let mut out = [0u8; $n];
406 for i in (0..$n).step_by(2) {
407 out[i..i + 2]
408 .copy_from_slice(&(u16at(&a, i).wrapping_mul(u16at(&b, i))).to_le_bytes());
409 }
410 out
411 }
412 #[inline(always)]
413 fn mulhiu16(self, a: Self::V, b: Self::V) -> Self::V {
414 let mut out = [0u8; $n];
415 for i in (0..$n).step_by(2) {
416 out[i..i + 2].copy_from_slice(
417 &(((u16at(&a, i) as u32 * u16at(&b, i) as u32) >> 16) as u16).to_le_bytes(),
418 );
419 }
420 out
421 }
422 #[inline(always)]
423 fn mulhis16(self, a: Self::V, b: Self::V) -> Self::V {
424 let mut out = [0u8; $n];
425 for i in (0..$n).step_by(2) {
426 out[i..i + 2].copy_from_slice(
427 &(((u16at(&a, i) as i16 as i32 * u16at(&b, i) as i16 as i32) >> 16) as u16)
428 .to_le_bytes(),
429 );
430 }
431 out
432 }
433 #[inline(always)]
434 fn mins16(self, a: Self::V, b: Self::V) -> Self::V {
435 let mut out = [0u8; $n];
436 for i in (0..$n).step_by(2) {
437 out[i..i + 2].copy_from_slice(
438 &((u16at(&a, i) as i16).min(u16at(&b, i) as i16) as u16).to_le_bytes(),
439 );
440 }
441 out
442 }
443 #[inline(always)]
444 fn maxs16(self, a: Self::V, b: Self::V) -> Self::V {
445 let mut out = [0u8; $n];
446 for i in (0..$n).step_by(2) {
447 out[i..i + 2].copy_from_slice(
448 &((u16at(&a, i) as i16).max(u16at(&b, i) as i16) as u16).to_le_bytes(),
449 );
450 }
451 out
452 }
453 #[inline(always)]
454 fn cmpgts16(self, a: Self::V, b: Self::V) -> Self::V {
455 let mut out = [0u8; $n];
456 for i in (0..$n).step_by(2) {
457 out[i..i + 2].copy_from_slice(
458 &(if (u16at(&a, i) as i16) > (u16at(&b, i) as i16) {
459 u16::MAX
460 } else {
461 0
462 })
463 .to_le_bytes(),
464 );
465 }
466 out
467 }
468 #[inline(always)]
469 fn srl16<const N: i32>(self, a: Self::V) -> Self::V {
470 let mut out = [0u8; $n];
471 for i in (0..$n).step_by(2) {
472 out[i..i + 2].copy_from_slice(
473 &(if (N as u32) < 16 {
474 u16at(&a, i) >> (N as u32)
475 } else {
476 0
477 })
478 .to_le_bytes(),
479 );
480 }
481 out
482 }
483 #[inline(always)]
484 fn sll16<const N: i32>(self, a: Self::V) -> Self::V {
485 let mut out = [0u8; $n];
486 for i in (0..$n).step_by(2) {
487 out[i..i + 2].copy_from_slice(
488 &(if (N as u32) < 16 {
489 u16at(&a, i) << (N as u32)
490 } else {
491 0
492 })
493 .to_le_bytes(),
494 );
495 }
496 out
497 }
498 #[inline(always)]
499 fn sra16<const N: i32>(self, a: Self::V) -> Self::V {
500 let mut out = [0u8; $n];
501 for i in (0..$n).step_by(2) {
502 out[i..i + 2].copy_from_slice(
503 &(((u16at(&a, i) as i16) >> (N as u32).min(15)) as u16).to_le_bytes(),
504 );
505 }
506 out
507 }
508 #[inline(always)]
509 fn cmpgts32(self, a: Self::V, b: Self::V) -> Self::V {
510 let mut out = [0u8; $n];
511 for i in (0..$n).step_by(4) {
512 out[i..i + 4].copy_from_slice(
513 &(if (u32at(&a, i) as i32) > (u32at(&b, i) as i32) {
514 u32::MAX
515 } else {
516 0
517 })
518 .to_le_bytes(),
519 );
520 }
521 out
522 }
523 #[inline(always)]
524 fn srl32<const N: i32>(self, a: Self::V) -> Self::V {
525 let mut out = [0u8; $n];
526 for i in (0..$n).step_by(4) {
527 out[i..i + 4].copy_from_slice(
528 &(if (N as u32) < 32 {
529 u32at(&a, i) >> (N as u32)
530 } else {
531 0
532 })
533 .to_le_bytes(),
534 );
535 }
536 out
537 }
538 #[inline(always)]
539 fn sll32<const N: i32>(self, a: Self::V) -> Self::V {
540 let mut out = [0u8; $n];
541 for i in (0..$n).step_by(4) {
542 out[i..i + 4].copy_from_slice(
543 &(if (N as u32) < 32 {
544 u32at(&a, i) << (N as u32)
545 } else {
546 0
547 })
548 .to_le_bytes(),
549 );
550 }
551 out
552 }
553 #[inline(always)]
554 fn sra32<const N: i32>(self, a: Self::V) -> Self::V {
555 let mut out = [0u8; $n];
556 for i in (0..$n).step_by(4) {
557 out[i..i + 4].copy_from_slice(
558 &(((u32at(&a, i) as i32) >> (N as u32).min(31)) as u32).to_le_bytes(),
559 );
560 }
561 out
562 }
563 #[inline(always)]
564 fn srl16x(self, a: Self::V, n: u32) -> Self::V {
565 let mut out = [0u8; $n];
566 for i in (0..$n).step_by(2) {
567 out[i..i + 2].copy_from_slice(
568 &(if n < 16 { u16at(&a, i) >> n } else { 0 }).to_le_bytes(),
569 );
570 }
571 out
572 }
573 #[inline(always)]
574 fn movemask8(self, a: Self::V) -> u32 {
575 let mut out = 0;
576 for i in (0..$n).step_by(1) {
577 out |= ((a[i] >> 7) as u32) << (i / 1);
578 }
579 out
580 }
581 #[inline(always)]
582 fn movemask32(self, a: Self::V) -> u32 {
583 let mut out = 0;
584 for i in (0..$n).step_by(4) {
585 out |= ((u32at(&a, i) >> 31) as u32) << (i / 4);
586 }
587 out
588 }
589 #[inline(always)]
590 fn unpacklo8(self, a: Self::V, b: Self::V) -> Self::V {
591 let mut out = [0; $n];
592 for lane in (0..$n).step_by(16) {
593 for j in (0..8).step_by(1) {
594 let dst = lane + j * 2;
595 let src = lane + 0 + j;
596 out[dst..dst + 1].copy_from_slice(&a[src..src + 1]);
597 out[dst + 1..dst + 2].copy_from_slice(&b[src..src + 1]);
598 }
599 }
600 out
601 }
602 #[inline(always)]
603 fn unpackhi8(self, a: Self::V, b: Self::V) -> Self::V {
604 let mut out = [0; $n];
605 for lane in (0..$n).step_by(16) {
606 for j in (0..8).step_by(1) {
607 let dst = lane + j * 2;
608 let src = lane + 8 + j;
609 out[dst..dst + 1].copy_from_slice(&a[src..src + 1]);
610 out[dst + 1..dst + 2].copy_from_slice(&b[src..src + 1]);
611 }
612 }
613 out
614 }
615 #[inline(always)]
616 fn unpacklo16(self, a: Self::V, b: Self::V) -> Self::V {
617 let mut out = [0; $n];
618 for lane in (0..$n).step_by(16) {
619 for j in (0..8).step_by(2) {
620 let dst = lane + j * 2;
621 let src = lane + 0 + j;
622 out[dst..dst + 2].copy_from_slice(&a[src..src + 2]);
623 out[dst + 2..dst + 4].copy_from_slice(&b[src..src + 2]);
624 }
625 }
626 out
627 }
628 #[inline(always)]
629 fn unpackhi16(self, a: Self::V, b: Self::V) -> Self::V {
630 let mut out = [0; $n];
631 for lane in (0..$n).step_by(16) {
632 for j in (0..8).step_by(2) {
633 let dst = lane + j * 2;
634 let src = lane + 8 + j;
635 out[dst..dst + 2].copy_from_slice(&a[src..src + 2]);
636 out[dst + 2..dst + 4].copy_from_slice(&b[src..src + 2]);
637 }
638 }
639 out
640 }
641 #[inline(always)]
642 fn packus16(self, a: Self::V, b: Self::V) -> Self::V {
643 let mut out = [0; $n];
644 for lane in (0..$n).step_by(16) {
645 for half in 0..2 {
646 let src = if half == 0 { &a } else { &b };
647 for j in 0..8 {
648 let i = lane + half * 8 + j * 1;
649 out[i] = (u16at(src, lane + j * 2) as i16).clamp(0, 255) as u8;
650 }
651 }
652 }
653 out
654 }
655 #[inline(always)]
656 fn packs16(self, a: Self::V, b: Self::V) -> Self::V {
657 let mut out = [0; $n];
658 for lane in (0..$n).step_by(16) {
659 for half in 0..2 {
660 let src = if half == 0 { &a } else { &b };
661 for j in 0..8 {
662 let i = lane + half * 8 + j * 1;
663 out[i] = (u16at(src, lane + j * 2) as i16).clamp(-128, 127) as u8;
664 }
665 }
666 }
667 out
668 }
669 #[inline(always)]
670 fn packs32(self, a: Self::V, b: Self::V) -> Self::V {
671 let mut out = [0; $n];
672 for lane in (0..$n).step_by(16) {
673 for half in 0..2 {
674 let src = if half == 0 { &a } else { &b };
675 for j in 0..4 {
676 let i = lane + half * 8 + j * 2;
677 out[i..i + 2].copy_from_slice(
678 &((u32at(src, lane + j * 4) as i32).clamp(-32768, 32767) as u16)
679 .to_le_bytes(),
680 );
681 }
682 }
683 }
684 out
685 }
686 #[inline(always)]
687 fn packssdw(self, a: Self::V, b: Self::V) -> Self::V {
688 self.packs32(a, b)
689 }
690 #[inline(always)]
691 fn packuswb_px(self, v: Self::V) -> Self::V {
692 let mut out = [0; $n];
693 for i in 0..$n / 2 {
694 out[i] = (u16at(&v, i * 2) as i16).clamp(0, 255) as u8;
695 }
696 for i in 0..$n / 2 {
697 out[$n / 2 + i] = out[i % 8];
698 }
699 out
700 }
701 #[inline(always)]
702 fn madd16(self, a: Self::V, b: Self::V) -> Self::V {
703 let mut out = [0u8; $n];
704 for i in (0..$n).step_by(4) {
705 out[i..i + 4].copy_from_slice(
706 &((u16at(&a, i) as i16 as i32)
707 .wrapping_mul(u16at(&b, i) as i16 as i32)
708 .wrapping_add((u16at(&a, i + 2) as i16 as i32).wrapping_mul(u16at(
709 &b,
710 i + 2,
711 )
712 as i16
713 as i32)) as u32)
714 .to_le_bytes(),
715 );
716 }
717 out
718 }
719 #[inline(always)]
720 fn shuf16<const IMM: i32>(self, a: Self::V) -> Self::V {
721 let mut out = [0; $n];
722 for half in (0..$n).step_by(8) {
723 for j in 0..4 {
724 let src = half + ((IMM as usize >> (j * 2)) & 3) * 2;
725 out[half + j * 2..half + j * 2 + 2].copy_from_slice(&a[src..src + 2]);
726 }
727 }
728 out
729 }
730 #[inline(always)]
731 fn shuf32<const IMM: i32>(self, a: Self::V) -> Self::V {
732 let mut out = [0; $n];
733 for lane in (0..$n).step_by(16) {
734 for j in 0..4 {
735 let src = lane + ((IMM as usize >> (j * 2)) & 3) * 4;
736 out[lane + j * 4..lane + j * 4 + 4].copy_from_slice(&a[src..src + 4]);
737 }
738 }
739 out
740 }
741 #[inline(always)]
742 fn shufps<const IMM: i32>(self, a: Self::V, b: Self::V) -> Self::V {
743 let mut out = [0; $n];
744 for lane in (0..$n).step_by(16) {
745 for j in 0..4 {
746 let src = lane + ((IMM as usize >> (j * 2)) & 3) * 4;
747 let from = if j < 2 { &a } else { &b };
748 out[lane + j * 4..lane + j * 4 + 4].copy_from_slice(&from[src..src + 4]);
749 }
750 }
751 out
752 }
753 #[inline(always)]
754 fn bsll<const N: i32>(self, a: Self::V) -> Self::V {
755 let mut out = [0; $n];
756 let n = N as usize;
757 if n < 16 {
758 for lane in (0..$n).step_by(16) {
759 out[lane + n..lane + 16].copy_from_slice(&a[lane..lane + 16 - n]);
760 }
761 }
762 out
763 }
764 #[inline(always)]
765 fn bsrl<const N: i32>(self, a: Self::V) -> Self::V {
766 let mut out = [0; $n];
767 let n = N as usize;
768 if n < 16 {
769 for lane in (0..$n).step_by(16) {
770 out[lane..lane + 16 - n].copy_from_slice(&a[lane + n..lane + 16]);
771 }
772 }
773 out
774 }
775 #[inline(always)]
776 fn and(self, a: Self::V, b: Self::V) -> Self::V {
777 let mut out = [0u8; $n];
778 for i in (0..$n).step_by(1) {
779 out[i] = a[i] & b[i];
780 }
781 out
782 }
783 #[inline(always)]
784 fn andnot(self, a: Self::V, b: Self::V) -> Self::V {
785 let mut out = [0u8; $n];
786 for i in (0..$n).step_by(1) {
787 out[i] = !a[i] & b[i];
788 }
789 out
790 }
791 #[inline(always)]
792 fn or(self, a: Self::V, b: Self::V) -> Self::V {
793 let mut out = [0u8; $n];
794 for i in (0..$n).step_by(1) {
795 out[i] = a[i] | b[i];
796 }
797 out
798 }
799 #[inline(always)]
800 fn xor(self, a: Self::V, b: Self::V) -> Self::V {
801 let mut out = [0u8; $n];
802 for i in (0..$n).step_by(1) {
803 out[i] = a[i] ^ b[i];
804 }
805 out
806 }
807 #[inline(always)]
808 fn addf(self, a: Self::V, b: Self::V) -> Self::V {
809 let mut out = [0u8; $n];
810 for i in (0..$n).step_by(4) {
811 out[i..i + 4].copy_from_slice(
812 &(portable_fx::arithmetic32(
813 f32::from_bits(u32at(&a, i)),
814 f32::from_bits(u32at(&b, i)),
815 f32::from_bits(u32at(&a, i)) + f32::from_bits(u32at(&b, i)),
816 )
817 .to_bits())
818 .to_le_bytes(),
819 );
820 }
821 out
822 }
823 #[inline(always)]
824 fn subf(self, a: Self::V, b: Self::V) -> Self::V {
825 let mut out = [0u8; $n];
826 for i in (0..$n).step_by(4) {
827 out[i..i + 4].copy_from_slice(
828 &(portable_fx::arithmetic32(
829 f32::from_bits(u32at(&a, i)),
830 f32::from_bits(u32at(&b, i)),
831 f32::from_bits(u32at(&a, i)) - f32::from_bits(u32at(&b, i)),
832 )
833 .to_bits())
834 .to_le_bytes(),
835 );
836 }
837 out
838 }
839 #[inline(always)]
840 fn mulf(self, a: Self::V, b: Self::V) -> Self::V {
841 let mut out = [0u8; $n];
842 for i in (0..$n).step_by(4) {
843 out[i..i + 4].copy_from_slice(
844 &(portable_fx::arithmetic32(
845 f32::from_bits(u32at(&a, i)),
846 f32::from_bits(u32at(&b, i)),
847 f32::from_bits(u32at(&a, i)) * f32::from_bits(u32at(&b, i)),
848 )
849 .to_bits())
850 .to_le_bytes(),
851 );
852 }
853 out
854 }
855 #[inline(always)]
856 fn divf(self, a: Self::V, b: Self::V) -> Self::V {
857 let mut out = [0u8; $n];
858 for i in (0..$n).step_by(4) {
859 out[i..i + 4].copy_from_slice(
860 &(portable_fx::arithmetic32(
861 f32::from_bits(u32at(&a, i)),
862 f32::from_bits(u32at(&b, i)),
863 f32::from_bits(u32at(&a, i)) / f32::from_bits(u32at(&b, i)),
864 )
865 .to_bits())
866 .to_le_bytes(),
867 );
868 }
869 out
870 }
871 #[inline(always)]
872 fn maxf(self, a: Self::V, b: Self::V) -> Self::V {
873 let mut out = [0u8; $n];
874 for i in (0..$n).step_by(4) {
875 out[i..i + 4].copy_from_slice(
876 &(if f32::from_bits(u32at(&a, i)) > f32::from_bits(u32at(&b, i)) {
877 u32at(&a, i)
878 } else {
879 u32at(&b, i)
880 })
881 .to_le_bytes(),
882 );
883 }
884 out
885 }
886 #[inline(always)]
887 fn minf(self, a: Self::V, b: Self::V) -> Self::V {
888 let mut out = [0u8; $n];
889 for i in (0..$n).step_by(4) {
890 out[i..i + 4].copy_from_slice(
891 &(if f32::from_bits(u32at(&a, i)) < f32::from_bits(u32at(&b, i)) {
892 u32at(&a, i)
893 } else {
894 u32at(&b, i)
895 })
896 .to_le_bytes(),
897 );
898 }
899 out
900 }
901 #[inline(always)]
902 fn cmpltf(self, a: Self::V, b: Self::V) -> Self::V {
903 let mut out = [0u8; $n];
904 for i in (0..$n).step_by(4) {
905 out[i..i + 4].copy_from_slice(
906 &(if f32::from_bits(u32at(&a, i)) < f32::from_bits(u32at(&b, i)) {
907 u32::MAX
908 } else {
909 0
910 })
911 .to_le_bytes(),
912 );
913 }
914 out
915 }
916 #[inline(always)]
917 fn sqrtf(self, a: Self::V) -> Self::V {
918 let mut out = [0u8; $n];
919 for i in (0..$n).step_by(4) {
920 out[i..i + 4].copy_from_slice(
921 &(rt::fx::sqrt(f32::from_bits(u32at(&a, i))).to_bits()).to_le_bytes(),
922 );
923 }
924 out
925 }
926 #[inline(always)]
927 fn addd(self, a: Self::V, b: Self::V) -> Self::V {
928 let mut out = [0u8; $n];
929 for i in (0..$n).step_by(8) {
930 out[i..i + 8].copy_from_slice(
931 &(portable_fx::arithmetic64(
932 f64::from_bits(u64at(&a, i)),
933 f64::from_bits(u64at(&b, i)),
934 f64::from_bits(u64at(&a, i)) + f64::from_bits(u64at(&b, i)),
935 )
936 .to_bits())
937 .to_le_bytes(),
938 );
939 }
940 out
941 }
942 #[inline(always)]
943 fn subd(self, a: Self::V, b: Self::V) -> Self::V {
944 let mut out = [0u8; $n];
945 for i in (0..$n).step_by(8) {
946 out[i..i + 8].copy_from_slice(
947 &(portable_fx::arithmetic64(
948 f64::from_bits(u64at(&a, i)),
949 f64::from_bits(u64at(&b, i)),
950 f64::from_bits(u64at(&a, i)) - f64::from_bits(u64at(&b, i)),
951 )
952 .to_bits())
953 .to_le_bytes(),
954 );
955 }
956 out
957 }
958 #[inline(always)]
959 fn muld(self, a: Self::V, b: Self::V) -> Self::V {
960 let mut out = [0u8; $n];
961 for i in (0..$n).step_by(8) {
962 out[i..i + 8].copy_from_slice(
963 &(portable_fx::arithmetic64(
964 f64::from_bits(u64at(&a, i)),
965 f64::from_bits(u64at(&b, i)),
966 f64::from_bits(u64at(&a, i)) * f64::from_bits(u64at(&b, i)),
967 )
968 .to_bits())
969 .to_le_bytes(),
970 );
971 }
972 out
973 }
974 #[inline(always)]
975 fn divd(self, a: Self::V, b: Self::V) -> Self::V {
976 let mut out = [0u8; $n];
977 for i in (0..$n).step_by(8) {
978 out[i..i + 8].copy_from_slice(
979 &(portable_fx::arithmetic64(
980 f64::from_bits(u64at(&a, i)),
981 f64::from_bits(u64at(&b, i)),
982 f64::from_bits(u64at(&a, i)) / f64::from_bits(u64at(&b, i)),
983 )
984 .to_bits())
985 .to_le_bytes(),
986 );
987 }
988 out
989 }
990 #[inline(always)]
991 fn maxd(self, a: Self::V, b: Self::V) -> Self::V {
992 let mut out = [0u8; $n];
993 for i in (0..$n).step_by(8) {
994 out[i..i + 8].copy_from_slice(
995 &(if f64::from_bits(u64at(&a, i)) > f64::from_bits(u64at(&b, i)) {
996 u64at(&a, i)
997 } else {
998 u64at(&b, i)
999 })
1000 .to_le_bytes(),
1001 );
1002 }
1003 out
1004 }
1005 #[inline(always)]
1006 fn mind(self, a: Self::V, b: Self::V) -> Self::V {
1007 let mut out = [0u8; $n];
1008 for i in (0..$n).step_by(8) {
1009 out[i..i + 8].copy_from_slice(
1010 &(if f64::from_bits(u64at(&a, i)) < f64::from_bits(u64at(&b, i)) {
1011 u64at(&a, i)
1012 } else {
1013 u64at(&b, i)
1014 })
1015 .to_le_bytes(),
1016 );
1017 }
1018 out
1019 }
1020 #[inline(always)]
1021 fn cmpltd(self, a: Self::V, b: Self::V) -> Self::V {
1022 let mut out = [0u8; $n];
1023 for i in (0..$n).step_by(8) {
1024 out[i..i + 8].copy_from_slice(
1025 &(if f64::from_bits(u64at(&a, i)) < f64::from_bits(u64at(&b, i)) {
1026 u64::MAX
1027 } else {
1028 0
1029 })
1030 .to_le_bytes(),
1031 );
1032 }
1033 out
1034 }
1035 #[inline(always)]
1036 fn sqrtd(self, a: Self::V) -> Self::V {
1037 let mut out = [0u8; $n];
1038 for i in (0..$n).step_by(8) {
1039 out[i..i + 8].copy_from_slice(
1040 &(rt::fx::sqrtd(f64::from_bits(u64at(&a, i))).to_bits()).to_le_bytes(),
1041 );
1042 }
1043 out
1044 }
1045 #[inline(always)]
1046 fn cvt_i32_f32(self, a: Self::V) -> Self::V {
1047 let mut out = [0u8; $n];
1048 for i in (0..$n).step_by(4) {
1049 out[i..i + 4]
1050 .copy_from_slice(&((u32at(&a, i) as i32 as f32).to_bits()).to_le_bytes());
1051 }
1052 out
1053 }
1054 #[inline(always)]
1055 fn cvtt_f32_i32(self, a: Self::V) -> Self::V {
1056 let mut out = [0u8; $n];
1057 for i in (0..$n).step_by(4) {
1058 out[i..i + 4].copy_from_slice(
1059 &(rt::fx::cvtt(f32::from_bits(u32at(&a, i))) as u32).to_le_bytes(),
1060 );
1061 }
1062 out
1063 }
1064 #[inline(always)]
1065 fn cvtr_f32_i32(self, a: Self::V) -> Self::V {
1066 let mut out = [0u8; $n];
1067 for i in (0..$n).step_by(4) {
1068 out[i..i + 4].copy_from_slice(
1069 &(rt::fx::cvt(f32::from_bits(u32at(&a, i))) as u32).to_le_bytes(),
1070 );
1071 }
1072 out
1073 }
1074 #[inline(always)]
1075 fn cvt_i32_f64(self, a: Self::V) -> Self::V {
1076 let mut out = [0u8; $n];
1077 for i in (0..$n).step_by(8) {
1078 out[i..i + 8].copy_from_slice(
1079 &((u32at(&a, i / 2) as i32 as f64).to_bits()).to_le_bytes(),
1080 );
1081 }
1082 out
1083 }
1084 #[inline(always)]
1085 fn cvtt_f64_i32(self, a: Self::V) -> Self::V {
1086 let mut out = [0; $n];
1087 for i in 0..$n / 8 {
1088 out[i * 4..i * 4 + 4].copy_from_slice(
1089 &rt::fx::cvttd(f64::from_bits(u64at(&a, i * 8))).to_le_bytes(),
1090 );
1091 }
1092 out
1093 }
1094 #[inline(always)]
1095 fn store_lanes2(self, b: &mut [u8], lo: Self::V, hi: Self::V) -> () {
1096 for lane in 0..$n / 16 {
1097 b[lane * 32..lane * 32 + 16].copy_from_slice(&lo[lane * 16..lane * 16 + 16]);
1098 b[lane * 32 + 16..lane * 32 + 32]
1099 .copy_from_slice(&hi[lane * 16..lane * 16 + 16]);
1100 }
1101 }
1102 #[inline(always)]
1103 fn load_lanes2(self, b: &[u8]) -> (Self::V, Self::V) {
1104 let mut lo = [0; $n];
1105 let mut hi = [0; $n];
1106 for lane in 0..$n / 16 {
1107 lo[lane * 16..lane * 16 + 16].copy_from_slice(&b[lane * 32..lane * 32 + 16]);
1108 hi[lane * 16..lane * 16 + 16]
1109 .copy_from_slice(&b[lane * 32 + 16..lane * 32 + 32]);
1110 }
1111 (lo, hi)
1112 }
1113 #[inline(always)]
1114 fn load_i16_lanes(self, a: &[i16], b: &[i16]) -> Self::V {
1115 let mut out = [0; $n];
1116 for lane in 0..$n / 16 {
1117 let from = if lane == 0 { a } else { b };
1118 for i in 0..8 {
1119 let dst = lane * 16 + i * 2;
1120 out[dst..dst + 2].copy_from_slice(&from[i].to_le_bytes());
1121 }
1122 }
1123 out
1124 }
1125 #[inline(always)]
1126 #[allow(clippy::out_of_bounds_indexing)] fn store_raw_lanes(self, a: &mut [u8], b: &mut [u8], v: Self::V) -> () {
1128 a[..8].copy_from_slice(&v[..8]);
1129 if $n > 16 {
1130 b[..8].copy_from_slice(&v[16..24]);
1131 }
1132 }
1133 #[inline(always)]
1134 fn load_f32_lanes(self, a: &[f32], b: &[f32]) -> Self::V {
1135 let mut out = [0; $n];
1136 for lane in 0..$n / 16 {
1137 let from = if lane == 0 { a } else { b };
1138 for i in 0..4 {
1139 let dst = lane * 16 + i * 4;
1140 out[dst..dst + 4].copy_from_slice(&from[i].to_bits().to_le_bytes());
1141 }
1142 }
1143 out
1144 }
1145 #[inline(always)]
1146 fn store_f32_lanes(self, a: &mut [f32], b: &mut [f32], v: Self::V) -> () {
1147 for i in 0..4 {
1148 a[i] = f32::from_bits(u32at(&v, i * 4));
1149 }
1150 if $n > 16 {
1151 for i in 0..4 {
1152 b[i] = f32::from_bits(u32at(&v, 16 + i * 4));
1153 }
1154 }
1155 }
1156 #[inline(always)]
1157 fn sadu8(self, a: Self::V, b: Self::V) -> Self::V {
1158 let mut out = [0; $n];
1159 for lane in (0..$n).step_by(8) {
1160 let mut sum = 0u64;
1161 for i in 0..8 {
1162 sum += a[lane + i].abs_diff(b[lane + i]) as u64;
1163 }
1164 out[lane..lane + 8].copy_from_slice(&sum.to_le_bytes());
1165 }
1166 out
1167 }
1168 #[inline(always)]
1169 fn ptest_raw(self, a: Self::V, b: Self::V) -> u32 {
1170 let mut z = 0;
1171 let mut c = 0;
1172 for i in 0..$n / 2 {
1173 z |= a[i] & b[i];
1174 c |= !a[i] & b[i];
1175 }
1176 if z == 0 {
1177 1
1178 } else if c == 0 {
1179 2
1180 } else {
1181 0
1182 }
1183 }
1184 #[inline(always)]
1185 fn ptestz_raw(self, a: Self::V, b: Self::V) -> bool {
1186 self.ptest_raw(a, b) == 1
1187 }
1188 }
1189 };
1190}
1191backend!(Scalar, 16);
1192backend!(Scalar256, 32);