1#[cfg(all(target_arch = "aarch64", not(feature = "no-std")))]
24use std::arch::is_aarch64_feature_detected;
25
26#[cfg(all(target_arch = "aarch64", feature = "no-std"))]
28use core::arch::aarch64;
29#[cfg(all(target_arch = "aarch64", not(feature = "no-std")))]
30use std::arch::aarch64;
31
32#[derive(Debug, Clone, Copy, PartialEq, Eq)]
34pub struct SimdCapabilities {
35 pub sse2: bool,
36 pub sse3: bool,
37 pub sse41: bool,
38 pub sse42: bool,
39 pub avx: bool,
40 pub avx2: bool,
41 pub avx512f: bool,
42 pub fma: bool,
43 pub neon: bool,
44}
45
46impl SimdCapabilities {
47 pub fn platform_name(&self) -> &'static str {
49 if self.avx512f {
50 "AVX-512"
51 } else if self.avx2 {
52 "AVX2"
53 } else if self.avx {
54 "AVX"
55 } else if self.sse42 {
56 "SSE4.2"
57 } else if self.sse41 {
58 "SSE4.1"
59 } else if self.sse3 {
60 "SSE3"
61 } else if self.sse2 {
62 "SSE2"
63 } else if self.neon {
64 "NEON"
65 } else {
66 "Scalar"
67 }
68 }
69}
70
71pub fn detect_simd_capabilities() -> SimdCapabilities {
84 SimdCapabilities {
85 sse2: detect_sse2(),
86 sse3: detect_sse3(),
87 sse41: detect_sse41(),
88 sse42: detect_sse42(),
89 avx: detect_avx(),
90 avx2: detect_avx2(),
91 avx512f: detect_avx512f(),
92 fma: detect_fma(),
93 neon: detect_neon(),
94 }
95}
96
97pub fn simd_width_f32() -> usize {
110 #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
111 {
112 if crate::simd_feature_detected!("avx512f") {
113 return 16; } else if crate::simd_feature_detected!("avx2") {
115 return 8; } else if crate::simd_feature_detected!("sse2") {
117 return 4; }
119 }
120
121 #[cfg(all(target_arch = "aarch64", not(feature = "no-std")))]
122 {
123 if is_aarch64_feature_detected!("neon") {
124 return 4; }
126 }
127
128 1 }
130
131pub fn optimal_chunk_size(array_len: usize, min_chunk: Option<usize>) -> usize {
149 let simd_width = simd_width_f32().max(1);
150 let min_chunk = min_chunk.unwrap_or(simd_width);
151
152 let preferred_chunk = simd_width.max(min_chunk);
154
155 let small_threshold = preferred_chunk.max(16);
157 if array_len <= small_threshold {
158 return array_len;
159 }
160
161 let cache_line_f32 = 64 / 4; let optimal = cache_line_f32.max(preferred_chunk);
164
165 let aligned = ((optimal / simd_width).max(1)) * simd_width;
167 aligned.min(array_len)
168}
169
170#[derive(Debug, Clone, Copy)]
176pub struct F32x4 {
177 #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
178 inner: core::arch::x86_64::__m128,
179 #[cfg(target_arch = "aarch64")]
180 inner: aarch64::float32x4_t,
181 #[cfg(not(any(target_arch = "x86", target_arch = "x86_64", target_arch = "aarch64")))]
182 inner: [f32; 4],
183}
184
185#[derive(Debug, Clone, Copy)]
187pub struct F32x8 {
188 #[allow(dead_code)] #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
190 inner: core::arch::x86_64::__m256,
191 #[allow(dead_code)] #[cfg(not(any(target_arch = "x86", target_arch = "x86_64")))]
193 inner: [f32; 8],
194}
195
196#[derive(Debug, Clone, Copy)]
198pub struct F32x16 {
199 #[allow(dead_code)] #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
201 inner: core::arch::x86_64::__m512,
202 #[allow(dead_code)] #[cfg(not(any(target_arch = "x86", target_arch = "x86_64")))]
204 inner: [f32; 16],
205}
206
207impl F32x4 {
208 #[inline]
210 pub fn splat(value: f32) -> Self {
211 #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
212 {
213 unsafe {
214 Self {
215 inner: core::arch::x86_64::_mm_set1_ps(value),
216 }
217 }
218 }
219
220 #[cfg(target_arch = "aarch64")]
221 {
222 unsafe {
223 Self {
224 inner: aarch64::vdupq_n_f32(value),
225 }
226 }
227 }
228
229 #[cfg(not(any(target_arch = "x86", target_arch = "x86_64", target_arch = "aarch64")))]
230 {
231 Self { inner: [value; 4] }
232 }
233 }
234
235 #[inline]
237 pub fn new(a: f32, b: f32, c: f32, d: f32) -> Self {
238 #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
239 {
240 unsafe {
241 Self {
242 inner: core::arch::x86_64::_mm_setr_ps(a, b, c, d),
243 }
244 }
245 }
246
247 #[cfg(target_arch = "aarch64")]
248 {
249 unsafe {
250 let arr = [a, b, c, d];
251 Self {
252 inner: aarch64::vld1q_f32(arr.as_ptr()),
253 }
254 }
255 }
256
257 #[cfg(not(any(target_arch = "x86", target_arch = "x86_64", target_arch = "aarch64")))]
258 {
259 Self {
260 inner: [a, b, c, d],
261 }
262 }
263 }
264
265 #[inline]
272 pub unsafe fn load_aligned(ptr: *const f32) -> Self {
273 #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
274 {
275 Self {
276 inner: core::arch::x86_64::_mm_load_ps(ptr),
277 }
278 }
279
280 #[cfg(target_arch = "aarch64")]
281 {
282 Self {
283 inner: aarch64::vld1q_f32(ptr),
284 }
285 }
286
287 #[cfg(not(any(target_arch = "x86", target_arch = "x86_64", target_arch = "aarch64")))]
288 {
289 Self {
290 inner: [*ptr, *ptr.add(1), *ptr.add(2), *ptr.add(3)],
291 }
292 }
293 }
294
295 #[inline]
301 pub unsafe fn load_unaligned(ptr: *const f32) -> Self {
302 #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
303 {
304 Self {
305 inner: core::arch::x86_64::_mm_loadu_ps(ptr),
306 }
307 }
308
309 #[cfg(target_arch = "aarch64")]
310 {
311 Self {
312 inner: aarch64::vld1q_f32(ptr),
313 }
314 }
315
316 #[cfg(not(any(target_arch = "x86", target_arch = "x86_64", target_arch = "aarch64")))]
317 {
318 Self {
319 inner: [*ptr, *ptr.add(1), *ptr.add(2), *ptr.add(3)],
320 }
321 }
322 }
323
324 #[inline]
331 pub unsafe fn store_aligned(self, ptr: *mut f32) {
332 #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
333 {
334 core::arch::x86_64::_mm_store_ps(ptr, self.inner);
335 }
336
337 #[cfg(target_arch = "aarch64")]
338 {
339 aarch64::vst1q_f32(ptr, self.inner);
340 }
341
342 #[cfg(not(any(target_arch = "x86", target_arch = "x86_64", target_arch = "aarch64")))]
343 {
344 *ptr = self.inner[0];
345 *ptr.add(1) = self.inner[1];
346 *ptr.add(2) = self.inner[2];
347 *ptr.add(3) = self.inner[3];
348 }
349 }
350
351 #[inline]
357 pub unsafe fn store_unaligned(self, ptr: *mut f32) {
358 #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
359 {
360 core::arch::x86_64::_mm_storeu_ps(ptr, self.inner);
361 }
362
363 #[cfg(target_arch = "aarch64")]
364 {
365 aarch64::vst1q_f32(ptr, self.inner);
366 }
367
368 #[cfg(not(any(target_arch = "x86", target_arch = "x86_64", target_arch = "aarch64")))]
369 {
370 *ptr = self.inner[0];
371 *ptr.add(1) = self.inner[1];
372 *ptr.add(2) = self.inner[2];
373 *ptr.add(3) = self.inner[3];
374 }
375 }
376
377 #[inline]
379 fn add_impl(self, other: Self) -> Self {
380 #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
381 {
382 unsafe {
383 Self {
384 inner: core::arch::x86_64::_mm_add_ps(self.inner, other.inner),
385 }
386 }
387 }
388
389 #[cfg(target_arch = "aarch64")]
390 {
391 unsafe {
392 Self {
393 inner: aarch64::vaddq_f32(self.inner, other.inner),
394 }
395 }
396 }
397
398 #[cfg(not(any(target_arch = "x86", target_arch = "x86_64", target_arch = "aarch64")))]
399 {
400 Self {
401 inner: [
402 self.inner[0] + other.inner[0],
403 self.inner[1] + other.inner[1],
404 self.inner[2] + other.inner[2],
405 self.inner[3] + other.inner[3],
406 ],
407 }
408 }
409 }
410
411 #[inline]
413 fn mul_impl(self, other: Self) -> Self {
414 #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
415 {
416 unsafe {
417 Self {
418 inner: core::arch::x86_64::_mm_mul_ps(self.inner, other.inner),
419 }
420 }
421 }
422
423 #[cfg(target_arch = "aarch64")]
424 {
425 unsafe {
426 Self {
427 inner: aarch64::vmulq_f32(self.inner, other.inner),
428 }
429 }
430 }
431
432 #[cfg(not(any(target_arch = "x86", target_arch = "x86_64", target_arch = "aarch64")))]
433 {
434 Self {
435 inner: [
436 self.inner[0] * other.inner[0],
437 self.inner[1] * other.inner[1],
438 self.inner[2] * other.inner[2],
439 self.inner[3] * other.inner[3],
440 ],
441 }
442 }
443 }
444
445 #[inline]
447 pub fn horizontal_sum(self) -> f32 {
448 #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
449 {
450 unsafe {
451 let temp = core::arch::x86_64::_mm_add_ps(
452 self.inner,
453 core::arch::x86_64::_mm_movehl_ps(self.inner, self.inner),
454 );
455 let result = core::arch::x86_64::_mm_add_ps(
456 temp,
457 core::arch::x86_64::_mm_shuffle_ps(temp, temp, 0x01),
458 );
459 core::arch::x86_64::_mm_cvtss_f32(result)
460 }
461 }
462
463 #[cfg(target_arch = "aarch64")]
464 {
465 unsafe {
466 let sum2 = aarch64::vpadd_f32(
467 aarch64::vget_low_f32(self.inner),
468 aarch64::vget_high_f32(self.inner),
469 );
470 let sum1 = aarch64::vpadd_f32(sum2, sum2);
471 aarch64::vget_lane_f32(sum1, 0)
472 }
473 }
474
475 #[cfg(not(any(target_arch = "x86", target_arch = "x86_64", target_arch = "aarch64")))]
476 {
477 self.inner[0] + self.inner[1] + self.inner[2] + self.inner[3]
478 }
479 }
480
481 #[inline]
483 pub fn extract(self, index: usize) -> f32 {
484 assert!(index < 4, "Index out of bounds");
485
486 #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
487 {
488 unsafe {
489 match index {
490 0 => core::arch::x86_64::_mm_cvtss_f32(self.inner),
491 1 => core::arch::x86_64::_mm_cvtss_f32(core::arch::x86_64::_mm_shuffle_ps(
492 self.inner, self.inner, 0x01,
493 )),
494 2 => core::arch::x86_64::_mm_cvtss_f32(core::arch::x86_64::_mm_shuffle_ps(
495 self.inner, self.inner, 0x02,
496 )),
497 3 => core::arch::x86_64::_mm_cvtss_f32(core::arch::x86_64::_mm_shuffle_ps(
498 self.inner, self.inner, 0x03,
499 )),
500 _ => unreachable!(),
501 }
502 }
503 }
504
505 #[cfg(target_arch = "aarch64")]
506 {
507 unsafe {
508 match index {
509 0 => aarch64::vgetq_lane_f32(self.inner, 0),
510 1 => aarch64::vgetq_lane_f32(self.inner, 1),
511 2 => aarch64::vgetq_lane_f32(self.inner, 2),
512 3 => aarch64::vgetq_lane_f32(self.inner, 3),
513 _ => unreachable!(),
514 }
515 }
516 }
517
518 #[cfg(not(any(target_arch = "x86", target_arch = "x86_64", target_arch = "aarch64")))]
519 {
520 self.inner[index]
521 }
522 }
523}
524
525impl core::ops::Add for F32x4 {
526 type Output = Self;
527 #[inline]
528 fn add(self, other: Self) -> Self {
529 self.add_impl(other)
530 }
531}
532
533impl core::ops::Mul for F32x4 {
534 type Output = Self;
535 #[inline]
536 fn mul(self, other: Self) -> Self {
537 self.mul_impl(other)
538 }
539}
540
541#[inline]
546fn detect_sse2() -> bool {
547 #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
548 {
549 crate::simd_feature_detected!("sse2")
550 }
551 #[cfg(not(any(target_arch = "x86", target_arch = "x86_64")))]
552 {
553 false
554 }
555}
556
557#[inline]
558fn detect_sse3() -> bool {
559 #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
560 {
561 crate::simd_feature_detected!("sse3")
562 }
563 #[cfg(not(any(target_arch = "x86", target_arch = "x86_64")))]
564 {
565 false
566 }
567}
568
569#[inline]
570fn detect_sse41() -> bool {
571 #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
572 {
573 crate::simd_feature_detected!("sse4.1")
574 }
575 #[cfg(not(any(target_arch = "x86", target_arch = "x86_64")))]
576 {
577 false
578 }
579}
580
581#[inline]
582fn detect_sse42() -> bool {
583 #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
584 {
585 crate::simd_feature_detected!("sse4.2")
586 }
587 #[cfg(not(any(target_arch = "x86", target_arch = "x86_64")))]
588 {
589 false
590 }
591}
592
593#[inline]
594fn detect_avx() -> bool {
595 #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
596 {
597 crate::simd_feature_detected!("avx")
598 }
599 #[cfg(not(any(target_arch = "x86", target_arch = "x86_64")))]
600 {
601 false
602 }
603}
604
605#[inline]
606fn detect_avx2() -> bool {
607 #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
608 {
609 crate::simd_feature_detected!("avx2")
610 }
611 #[cfg(not(any(target_arch = "x86", target_arch = "x86_64")))]
612 {
613 false
614 }
615}
616
617#[inline]
618fn detect_avx512f() -> bool {
619 #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
620 {
621 crate::simd_feature_detected!("avx512f")
622 }
623 #[cfg(not(any(target_arch = "x86", target_arch = "x86_64")))]
624 {
625 false
626 }
627}
628
629#[inline]
630fn detect_fma() -> bool {
631 #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
632 {
633 crate::simd_feature_detected!("fma")
634 }
635 #[cfg(not(any(target_arch = "x86", target_arch = "x86_64")))]
636 {
637 false
638 }
639}
640
641#[inline]
642fn detect_neon() -> bool {
643 #[cfg(all(target_arch = "aarch64", not(feature = "no-std")))]
644 {
645 is_aarch64_feature_detected!("neon")
646 }
647 #[cfg(all(target_arch = "aarch64", feature = "no-std"))]
648 {
649 false
650 }
651 #[cfg(not(target_arch = "aarch64"))]
652 {
653 false
654 }
655}
656
657#[inline]
663pub fn is_aligned(ptr: *const u8, alignment: usize) -> bool {
664 if alignment == 0 || !alignment.is_power_of_two() {
665 return false;
666 }
667
668 (ptr as usize) & (alignment - 1) == 0
669}
670
671#[inline]
673pub fn align_up(value: usize, alignment: usize) -> usize {
674 if alignment == 0 {
675 return value;
676 }
677
678 let mask = alignment - 1;
679 if !alignment.is_power_of_two() {
680 return if value.is_multiple_of(alignment) {
682 value
683 } else {
684 value + (alignment - (value % alignment))
685 };
686 }
687
688 (value + mask) & !mask
689}
690
691#[inline]
693pub fn align_down(value: usize, alignment: usize) -> usize {
694 if alignment == 0 {
695 return value;
696 }
697
698 let mask = alignment - 1;
699 if !alignment.is_power_of_two() {
700 return value - (value % alignment);
701 }
702
703 value & !mask
704}
705
706#[inline]
708pub fn preferred_alignment_f32() -> usize {
709 #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
710 {
711 if detect_avx512f() {
712 64 } else if detect_avx2() {
714 32 } else if detect_sse2() {
716 16 } else {
718 4 }
720 }
721
722 #[cfg(target_arch = "aarch64")]
723 {
724 if detect_neon() {
725 16 } else {
727 4 }
729 }
730
731 #[cfg(not(any(target_arch = "x86", target_arch = "x86_64", target_arch = "aarch64")))]
732 {
733 4 }
735}
736
737#[allow(non_snake_case)]
738#[cfg(all(test, not(feature = "no-std")))]
739mod tests {
740 use super::*;
741
742 #[cfg(feature = "no-std")]
743 use alloc::{vec, vec::Vec};
744
745 #[test]
746 fn test_simd_capabilities() {
747 let caps = detect_simd_capabilities();
748
749 #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
751 assert!(caps.sse2 || caps.avx2 || caps.avx512f);
752
753 #[cfg(target_arch = "aarch64")]
754 assert!(caps.neon);
755
756 println!("SIMD capabilities: {:?}", caps);
757 }
758
759 #[test]
760 fn test_simd_width() {
761 let width = simd_width_f32();
762 assert!(width >= 1);
763 assert!(width <= 16);
764
765 #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
766 {
767 if detect_avx512f() {
768 assert_eq!(width, 16);
769 } else if detect_avx2() {
770 assert_eq!(width, 8);
771 } else if detect_sse2() {
772 assert_eq!(width, 4);
773 } else {
774 assert_eq!(width, 1);
775 }
776 }
777
778 #[cfg(target_arch = "aarch64")]
779 {
780 if detect_neon() {
781 assert_eq!(width, 4);
782 } else {
783 assert_eq!(width, 1);
784 }
785 }
786 }
787
788 #[test]
789 fn test_optimal_chunk_size() {
790 let small_chunk = optimal_chunk_size(10, None);
792 assert_eq!(small_chunk, 10);
793
794 let large_chunk = optimal_chunk_size(1000, None);
796 let simd_width = simd_width_f32();
797 assert!(large_chunk >= simd_width);
798 assert_eq!(large_chunk % simd_width, 0);
799
800 let min_chunk = optimal_chunk_size(1000, Some(32));
802 assert!(min_chunk >= 32);
803 }
804
805 #[test]
806 fn test_f32x4_basic_operations() {
807 let a = F32x4::new(1.0, 2.0, 3.0, 4.0);
808 let b = F32x4::new(5.0, 6.0, 7.0, 8.0);
809
810 assert_eq!(a.extract(0), 1.0);
812 assert_eq!(a.extract(1), 2.0);
813 assert_eq!(a.extract(2), 3.0);
814 assert_eq!(a.extract(3), 4.0);
815
816 let sum = a + b;
818 assert_eq!(sum.extract(0), 6.0);
819 assert_eq!(sum.extract(1), 8.0);
820 assert_eq!(sum.extract(2), 10.0);
821 assert_eq!(sum.extract(3), 12.0);
822
823 let product = a * b;
825 assert_eq!(product.extract(0), 5.0);
826 assert_eq!(product.extract(1), 12.0);
827 assert_eq!(product.extract(2), 21.0);
828 assert_eq!(product.extract(3), 32.0);
829
830 assert_eq!(a.horizontal_sum(), 10.0);
832 }
833
834 #[test]
835 fn test_f32x4_splat() {
836 let splat = F32x4::splat(42.0);
837
838 assert_eq!(splat.extract(0), 42.0);
839 assert_eq!(splat.extract(1), 42.0);
840 assert_eq!(splat.extract(2), 42.0);
841 assert_eq!(splat.extract(3), 42.0);
842
843 assert_eq!(splat.horizontal_sum(), 168.0); }
845
846 #[test]
847 fn test_f32x4_load_store() {
848 let data = vec![1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0];
849 let mut result = vec![0.0; 8];
850
851 unsafe {
852 let vec1 = F32x4::load_unaligned(data.as_ptr());
854 let vec2 = F32x4::load_unaligned(data.as_ptr().add(4));
855
856 assert_eq!(vec1.extract(0), 1.0);
858 assert_eq!(vec1.extract(1), 2.0);
859 assert_eq!(vec1.extract(2), 3.0);
860 assert_eq!(vec1.extract(3), 4.0);
861
862 vec1.store_unaligned(result.as_mut_ptr());
864 vec2.store_unaligned(result.as_mut_ptr().add(4));
865 }
866
867 assert_eq!(result, data);
868 }
869
870 #[test]
871 fn test_alignment_functions() {
872 #[repr(align(32))]
873 struct AlignedBytes([u8; 32]);
874
875 let aligned_storage = AlignedBytes([0u8; 32]);
876 let aligned_ptr = aligned_storage.0.as_ptr();
878 let unaligned_ptr = unsafe { aligned_ptr.add(1) };
879
880 assert!(is_aligned(aligned_ptr, 16));
881 assert!(!is_aligned(unaligned_ptr, 16));
882
883 assert_eq!(align_up(15, 16), 16);
885 assert_eq!(align_up(16, 16), 16);
886 assert_eq!(align_up(17, 16), 32);
887
888 assert_eq!(align_down(15, 16), 0);
889 assert_eq!(align_down(16, 16), 16);
890 assert_eq!(align_down(31, 16), 16);
891 }
892
893 #[test]
894 fn test_preferred_alignment() {
895 let alignment = preferred_alignment_f32();
896
897 assert!(alignment >= 4);
899 assert!(alignment.is_power_of_two());
900
901 #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
903 {
904 if detect_avx512f() {
905 assert_eq!(alignment, 64);
906 } else if detect_avx2() {
907 assert_eq!(alignment, 32);
908 } else if detect_sse2() {
909 assert_eq!(alignment, 16);
910 } else {
911 assert_eq!(alignment, 4);
912 }
913 }
914
915 #[cfg(target_arch = "aarch64")]
916 {
917 if detect_neon() {
918 assert_eq!(alignment, 16);
919 } else {
920 assert_eq!(alignment, 4);
921 }
922 }
923 }
924
925 #[test]
926 fn test_large_vector_operations() {
927 let size = 1000;
929 let data: Vec<f32> = (0..size).map(|i| i as f32).collect();
930 let mut result = vec![0.0; size];
931
932 let lane_width = 4; let chunks = size / lane_width;
934
935 for i in 0..chunks {
936 let offset = i * lane_width;
937 unsafe {
938 let vec = F32x4::load_unaligned(data.as_ptr().add(offset));
939 let doubled = vec + vec; doubled.store_unaligned(result.as_mut_ptr().add(offset));
941 }
942 }
943
944 for (i, &val) in result.iter().enumerate().take(chunks * lane_width) {
946 assert_eq!(val, 2.0 * (i as f32));
947 }
948 }
949
950 #[test]
951 #[should_panic(expected = "Index out of bounds")]
952 fn test_f32x4_extract_out_of_bounds() {
953 let vec = F32x4::splat(1.0);
954 vec.extract(4); }
956}