tract_linalg/x86_64/
min.rs

1routine_reduce_rust!(x86_64;
2    f32,
3    x86_64_fma_min_f32_32n,
4    32,
5    8,
6    #[inline(never)]
7    fn run(buf: &[f32], _: ()) -> f32 {
8        assert!(buf.len() % 32 == 0);
9        assert!(buf.len() > 0);
10        unsafe { x86_64_fma_min_f32_32n_run(buf) }
11    },
12    op(Min),
13    isa(X86_64Avx)
14);
15
16#[cfg(target_arch = "x86_64")]
17#[target_feature(enable = "avx")]
18unsafe fn x86_64_fma_min_f32_32n_run(buf: &[f32]) -> f32 {
19    unsafe {
20        let len = buf.len();
21        let ptr = buf.as_ptr();
22        let mut acc = f32::MAX;
23        std::arch::asm!("
24            // reg-source vbroadcastss needs avx2; this kernel must stay avx-safe
25            vpermilps xmm0, xmm0, 0
26            vinsertf128 ymm0, ymm0, xmm0, 1
27            vmovaps ymm1, ymm0
28            vmovaps ymm2, ymm0
29            vmovaps ymm3, ymm0
30            2:
31                vmovaps ymm4, [{ptr}]
32                vmovaps ymm5, [{ptr} + 32]
33                vmovaps ymm6, [{ptr} + 64]
34                vmovaps ymm7, [{ptr} + 96]
35                vminps ymm0, ymm0, ymm4
36                vminps ymm1, ymm1, ymm5
37                vminps ymm2, ymm2, ymm6
38                vminps ymm3, ymm3, ymm7
39                add {ptr}, 128
40                sub {len}, 32
41                jnz 2b
42            vminps ymm0, ymm0, ymm1
43            vminps ymm2, ymm2, ymm3
44            vminps ymm0, ymm0, ymm2
45            vperm2f128 ymm1, ymm0, ymm0, 1      // copy second half (4xf32) of ymm0 to ymm1
46            vminps xmm0, xmm0, xmm1             // xmm0 contains 4 values to min
47            vpermilps xmm1, xmm0, 2 + (3 << 2)  // second 2x32 bit half moved to top
48            vminps xmm0, xmm0, xmm1             // xmm0 containes 2 values
49            vpermilps xmm1, xmm0, 1             // second f32 to top
50            vminps xmm0, xmm0, xmm1
51            ",
52        len = inout(reg) len => _,
53        ptr = inout(reg) ptr => _,
54        inout("ymm0") acc,
55        out("ymm1") _, out("ymm2") _, out("ymm3") _,
56        out("ymm4") _, out("ymm5") _, out("ymm6") _, out("ymm7") _
57        );
58        acc
59    }
60}
tract_linalg/x86_64/min.rs

tract_linalg/x86_64/
min.rs
