thermite 0.2.1

High-performance, generic, ISA-portable SIMD library with a policy-configurable transcendental math library
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141

/// Reduces an 8-lane `f32` SIMD vector using two operations simultaneously, returning both results.
///
/// Splits the YMM into two XMM halves, applies both ops across the halves independently,
/// then delegates the 4-lane XMM reductions to `_mm_reduce_ps_v2!`.
#[rustfmt::skip]
macro_rules! _mm256_reduce2_ps_v3 {
    ($value:expr; $op1:ident $last1:ident, $op2:ident $last2:ident) => {#[allow(unused_unsafe)] unsafe {
        let ymm0 = $value;
        let xmm0 = arch::_mm256_castps256_ps128(ymm0);
        let xmm1 = arch::_mm256_extractf128_ps(ymm0, 1);
        let xmm_a = arch::$op1(xmm0, xmm1);
        let xmm_b = arch::$op2(xmm0, xmm1);
        (
            _mm_reduce_ps_v2!(xmm_a; $op1 $last1),
            _mm_reduce_ps_v2!(xmm_b; $op2 $last2),
        )
    }};
}

/// Reduces a 4-lane `f64` SIMD vector using two operations simultaneously, returning both results.
///
/// Splits the YMM into two XMM halves, applies both ops across the halves independently,
/// then delegates the 2-lane XMM reductions to `_mm_reduce_pd_v1!`.
#[rustfmt::skip]
macro_rules! _mm256_reduce2_pd_v3 {
    ($value:expr; $op1:ident $last1:ident, $op2:ident $last2:ident) => {#[allow(unused_unsafe)] unsafe {
        let ymm0 = $value;
        let xmm0 = arch::_mm256_castpd256_pd128(ymm0);
        let xmm1 = arch::_mm256_extractf128_pd(ymm0, 1);
        let xmm_a = arch::$op1(xmm0, xmm1);
        let xmm_b = arch::$op2(xmm0, xmm1);
        (
            _mm_reduce_pd_v1!(xmm_a; $op1 $last1),
            _mm_reduce_pd_v1!(xmm_b; $op2 $last2),
        )
    }};
}

/// Reduces an 8-lane `f32` SIMD vector to a single `f32` value using the specified operation.
#[rustfmt::skip]
macro_rules! _mm256_reduce_ps_v3 {
    ($value:expr; $op:ident $last:ident) => {#[allow(unused_unsafe)] unsafe {
        let ymm0 = $value;
        let xmm0 = arch::_mm256_castps256_ps128(ymm0);
        let xmm1 = arch::_mm256_extractf128_ps(ymm0, 1);

        let xmm0 = arch::$op(xmm0, xmm1);

        _mm_reduce_ps_v2!(xmm0; $op $last)
    }};
}

/// Reduces a 4-lane `f64` SIMD vector to a single `f64` value using the specified operation.
#[rustfmt::skip]
macro_rules! _mm256_reduce_pd_v3 {
    ($value:expr; $op:ident $last:ident) => {#[allow(unused_unsafe)] unsafe {
        let ymm0 = $value;
        let xmm0 = arch::_mm256_castpd256_pd128(ymm0);
        let xmm1 = arch::_mm256_extractf128_pd(ymm0, 1);

        let xmm0 = arch::$op(xmm0, xmm1);

        _mm_reduce_pd_v1!(xmm0; $op $last)
    }};
}

/// Reduces a 4-lane `f64` SIMD vector to a single `f64` value using the specified operation,
/// ignoring the last lane entirely.
#[rustfmt::skip]
macro_rules! _mm256_reduce_pd3_v3 {
    ($value:expr; $last:ident) => {#[allow(unused_unsafe)] unsafe {
        let ymm0 = $value;

        // [0, 1]
        let xmm0 = arch::_mm256_castpd256_pd128(ymm0);
        // [1, 0]
        let xmm1 = arch::_mm256_castpd256_pd128(arch::_mm256_permute4x64_pd(ymm0, 0b00_00_00_01));
        let xmm1 = arch::$last(xmm0, xmm1); // [0 + 1, 1]

        // [2, 0]
        let xmm2 = arch::_mm256_castpd256_pd128(arch::_mm256_permute4x64_pd(ymm0, 0b00_00_00_10));

        // [0 + 1 + 2, 1]
        arch::_mm_cvtsd_f64(arch::$last(xmm1, xmm2))

    }};
}

/// Reduces an 8-lane `i32` SIMD vector to a single `i32` value using the specified operation.
#[rustfmt::skip]
macro_rules! _mm256_reduce_epi32_v3 {
    ($value:expr; $op:ident $last:ident) => {{#[allow(unused_unsafe)] unsafe {
        let ymm0 = $value;
        let xmm0 = arch::_mm256_castsi256_si128(ymm0);
        let xmm1 = arch::_mm256_extractf128_si256(ymm0, 1);

        let xmm0 = arch::$op(xmm0, xmm1);

        _mm_reduce_epi32_v1!(xmm0; $op $last)
    }}};
}

/// Reduces a 16-lane 16-bit integer SIMD vector to a single lane value using `$op128` (the
/// 128-bit lane-wise op, e.g. `_mm_min_epi16`). Folds the two 128-bit halves, then delegates
/// to the 128-bit `_mm_reduce_epi16_v2!`. Returns an `i16` (cast at the call site for u16).
#[rustfmt::skip]
macro_rules! _mm256_reduce_epi16_v3 {
    ($value:expr; $op128:ident) => {{#[allow(unused_unsafe)] unsafe {
        let ymm0 = $value;
        let xmm0 = arch::_mm256_castsi256_si128(ymm0);
        let xmm1 = arch::_mm256_extracti128_si256(ymm0, 1);
        let xmm0 = arch::$op128(xmm0, xmm1); // fold high 8 lanes into low 8
        _mm_reduce_epi16_v2!(xmm0; $op128)
    }}};
}

/// Reduces a 32-lane 8-bit integer SIMD vector to a single lane value using `$op128` (the
/// 128-bit lane-wise op, e.g. `_mm_min_epi8`). Folds the two 128-bit halves, then delegates to
/// the 128-bit `_mm_reduce_epi8_v2!`. Returns an `i8` (cast at the call site for u8).
#[rustfmt::skip]
macro_rules! _mm256_reduce_epi8_v3 {
    ($value:expr; $op128:ident) => {{#[allow(unused_unsafe)] unsafe {
        let ymm0 = $value;
        let xmm0 = arch::_mm256_castsi256_si128(ymm0);
        let xmm1 = arch::_mm256_extracti128_si256(ymm0, 1);
        let xmm0 = arch::$op128(xmm0, xmm1); // fold high 16 lanes into low 16
        _mm_reduce_epi8_v2!(xmm0; $op128)
    }}};
}

macro_rules! _mm256_reduce_epi64_v3 {
    ($value:expr; $op:ident $last:ident) => {{#[allow(unused_unsafe)] unsafe {
        let ymm0 = $value;
        let xmm0 = arch::_mm256_castsi256_si128(ymm0);
        let xmm1 = arch::_mm256_extractf128_si256(ymm0, 1);

        let xmm0 = arch::$op(xmm0, xmm1);

        _mm_reduce_epi64_v1!(xmm0; $last)
    }}};
}