thermite 0.2.1

High-performance, generic, ISA-portable SIMD library with a policy-configurable transcendental math library
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116

use super::*;

// NOTE: Saturated add/sub use the "sign bit" as the select bit,
// so when porting to SSE2 it'll need to use the `signbits` methods to properly select with `_mm_blendv_epi8x`

// NOTE: `_mm_blendv_epi8` selects per **byte** using each byte's high bit, so
// every value used as a select mask must be sign-broadcast across the whole
// lane (all bytes equal) - a raw operand/result has arbitrary low-byte high
// bits and would saturate only some bytes of a lane, producing garbage. The
// `signbits` helpers do that broadcast; `_mm_cmpgt_*` results are already
// lane-uniform. The boolean logic itself is unchanged.

#[inline(always)]
pub unsafe fn _mm_adds_epi64x_v2(lhs: __m128i, rhs: __m128i) -> __m128i {
    let res = _mm_add_epi64(lhs, rhs);

    _mm_blendv_epi8(
        res,
        _mm_blendv_epi8(
            _mm_set1_epi64x(i64::MIN),
            _mm_set1_epi64x(i64::MAX),
            _mm_signbits_epi64x_v1(res),
        ),
        _mm_xor_si128(_mm_signbits_epi64x_v1(rhs), _mm_cmpgt_epi64(lhs, res)),
    )
}

#[inline(always)]
pub unsafe fn _mm_adds_epi32x_v2(lhs: __m128i, rhs: __m128i) -> __m128i {
    let res = _mm_add_epi32(lhs, rhs);

    _mm_blendv_epi8(
        res,
        _mm_blendv_epi8(
            _mm_set1_epi32(i32::MIN),
            _mm_set1_epi32(i32::MAX),
            _mm_signbits_epi32x_v1(res),
        ),
        _mm_xor_si128(_mm_signbits_epi32x_v1(rhs), _mm_cmpgt_epi32(lhs, res)),
    )
}

#[inline(always)]
pub unsafe fn _mm_subs_epi32x_v2(lhs: __m128i, rhs: __m128i) -> __m128i {
    let res = _mm_sub_epi32(lhs, rhs);

    _mm_blendv_epi8(
        res,
        _mm_blendv_epi8(
            _mm_set1_epi32(i32::MIN),
            _mm_set1_epi32(i32::MAX),
            _mm_signbits_epi32x_v1(res),
        ),
        _mm_xor_si128(_mm_cmpgt_epi32(rhs, _mm_setzero_si128()), _mm_cmpgt_epi32(lhs, res)),
    )
}

#[inline(always)]
pub unsafe fn _mm_subs_epi64x_v2(lhs: __m128i, rhs: __m128i) -> __m128i {
    let res = _mm_sub_epi64(lhs, rhs);

    _mm_blendv_epi8(
        res,
        _mm_blendv_epi8(
            _mm_set1_epi64x(i64::MIN),
            _mm_set1_epi64x(i64::MAX),
            _mm_signbits_epi64x_v1(res),
        ),
        _mm_xor_si128(_mm_cmpgt_epi64(rhs, _mm_setzero_si128()), _mm_cmpgt_epi64(lhs, res)),
    )
}

#[inline(always)]
pub unsafe fn zero4_v2(value: __m128) -> __m128 {
    // NOTE: Compiler may choose to replace the set1 with `xorps xmm, xmm`
    // this may be better than the SSE2 version that uses `andps` with a mask
    // for the last lane, simply because xorps can be paired with other instructions.
    unsafe { _mm_blend_ps(value, _mm_set1_ps(0.0), 0b1000) }
}

#[inline(always)]
pub unsafe fn one4_v2(value: __m128) -> __m128 {
    unsafe { _mm_blend_ps(value, _mm_set1_ps(1.0), 0b1000) }
}

// https://stackoverflow.com/a/76436268/2083075
#[inline(always)]
pub unsafe fn _mm_mullo_epi64x_v2(lhs: __m128i, rhs: __m128i) -> __m128i {
    let bswap = _mm_shuffle_epi32(rhs, 0xB1);
    let prodlh = _mm_mullo_epi32(lhs, bswap);

    let prodlh2 = _mm_srli_epi64(prodlh, 32);
    let prodlh3 = _mm_add_epi32(prodlh2, prodlh);
    let prodlh4 = _mm_and_si128(prodlh3, _mm_set1_epi64x(0x00000000FFFFFFFF));

    let prodll = _mm_mul_epu32(lhs, rhs);
    // The cross term (a_lo*b_hi + a_hi*b_lo) occupies bits 32..63, so it must
    // be shifted left by 32 before being added to the low product. The mask
    // above zeroes its high dword so the shift can't bleed garbage in.
    let prod = _mm_add_epi64(prodll, _mm_slli_epi64(prodlh4, 32));

    prod
}

#[inline(always)]
pub unsafe fn _mm_copysign_epi64x_v2(lhs: __m128i, rhs: __m128i) -> __m128i {
    let change_sign = _mm_xor_si128(
        _mm_cmpgt_epi64(rhs, _mm_set1_epi64x(-1)), // rhs > -1 = rhs >= 0
        _mm_cmpgt_epi64(lhs, _mm_set1_epi64x(-1)), // lhs > -1 = lhs >= 0
    );

    _mm_add_epi64(
        _mm_xor_si128(lhs, change_sign), // invert lhs if change_sign is true
        _mm_srli_epi64(change_sign, 63), // 1 if true, 0 if false, to correct for two's complement
    )
}