thermite 0.2.0

High-performance, generic, ISA-portable SIMD library with a policy-configurable transcendental math library
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118

#![allow(unsafe_op_in_unsafe_fn, clippy::missing_safety_doc, clippy::let_and_return)]

use super::arch::*;

pub use crate::backend::x86_v1::polyfills::*;

pub mod bits;
pub mod casts;
pub mod cmp;
pub mod divider;
pub mod math;

pub use bits::*;
pub use casts::*;
pub use cmp::*;
pub use divider::*;
pub use math::*;

/// This uses _mm_shuffle_epi8 to permute the register as that's the only
/// instruction for arbitrary register-controlled shuffles available to SSE4.1
/// effectively recreating _mm_permutevar_ps/_mm_permutevar_epi32
///
/// An alternative implementation is here: <https://stackoverflow.com/a/56033645/2083075>
/// but that uses _mm_mullo_epi32 which has a 10 cycle latency on some CPUs,
/// whereas these use only _mm_shuffle_epi8 and simple arithmetic/logical ops,
/// which are much faster.
#[inline(always)]
pub unsafe fn _mm_permutevarx_epi32x_v2(value: __m128i, indices: __m128i) -> __m128i {
    // ensure that all indices are valid by simply AND-ing them with 0b11
    let indices = _mm_and_si128(indices, _mm_set1_epi32(0x3));

    let shuffled_bases = _mm_shuffle_epi8(
        _mm_slli_epi32(indices, 2), // index * 4
        _mm_set_epi32(0x0C0C0C0C, 0x08080808, 0x04040404, 0x00000000),
    );

    // add 0,1,2,3 to each base index to get byte indices
    _mm_shuffle_epi8(value, _mm_add_epi8(shuffled_bases, _mm_set1_epi32(0x03020100)))
}

#[inline(always)]
pub unsafe fn _mm_permutevarx_epi64x_v2(value: __m128i, indices: __m128i) -> __m128i {
    // ensure that all indices are valid by simply AND-ing them with 0b1
    let indices = _mm_and_si128(indices, _mm_set1_epi64x(0x1));

    let shuffled_bases = _mm_shuffle_epi8(
        _mm_slli_epi64(indices, 3), // index * 8
        _mm_set_epi64x(0x0808080808080808, 0x0000000000000000),
    );

    // add 0..7 to each base index to get byte indices
    _mm_shuffle_epi8(value, _mm_add_epi8(shuffled_bases, _mm_set1_epi64x(0x0706050403020100)))
}

/// Variable within-register permute of 8x16-bit lanes: `result[i] = value[idx[i] & 7]`.
///
/// `idx_lo`/`idx_hi` are the 8 lane indices as two `u32x4` registers (the low and high four
/// of a `GenericArray<u32, 8>`). The word indices are narrowed to `u16` and expanded into a
/// byte-shuffle mask via the `w*0x0202 + 0x0100` identity (which places bytes `[2w, 2w+1]`
/// per lane), then applied with a single `pshufb`.
///
/// Out-of-range indices are undefined per the `permutev` contract; they are not masked here.
/// `pshufb` is still memory-safe (it clamps the byte index within the register), so an OOB
/// lane just yields an unspecified value rather than UB.
#[inline(always)]
pub unsafe fn _mm_permutev_epi16x_v2(value: __m128i, idx_lo: __m128i, idx_hi: __m128i) -> __m128i {
    let widx = _mm_packus_epi32(idx_lo, idx_hi); // narrow 8x u32 -> 8x u16
    let byte_mask = _mm_add_epi16(
        _mm_mullo_epi16(widx, _mm_set1_epi16(0x0202u16 as i16)),
        _mm_set1_epi16(0x0100),
    );
    _mm_shuffle_epi8(value, byte_mask)
}

/// Variable within-register permute of 16x8-bit lanes: `result[i] = value[idx[i] & 15]`.
///
/// The 16 lane indices arrive as four `u32x4` registers (a `GenericArray<u32, 16>`). For
/// 8-bit lanes the index *is* the byte index, so they are packed straight down to 16 `u8`
/// (`packus_epi32` then `packus_epi16`) and applied with a single `pshufb`.
///
/// Out-of-range indices are undefined per the `permutev` contract; they are not masked here.
/// `pshufb` remains memory-safe regardless (it uses only the low 4 bits when the high bit is
/// clear, and yields 0 when set), so an OOB lane just produces an unspecified value, not UB.
#[inline(always)]
pub unsafe fn _mm_permutev_epi8x_v2(value: __m128i, i0: __m128i, i1: __m128i, i2: __m128i, i3: __m128i) -> __m128i {
    let lo = _mm_packus_epi32(i0, i1); // 8x u16
    let hi = _mm_packus_epi32(i2, i3); // 8x u16
    let bytes = _mm_packus_epi16(lo, hi); // 16x u8 byte indices
    _mm_shuffle_epi8(value, bytes)
}

#[inline(always)]
pub unsafe fn _mm_permutevar_ps_v2(value: __m128, indices: __m128i) -> __m128 {
    _mm_castsi128_ps(_mm_permutevarx_epi32x_v2(_mm_castps_si128(value), indices))
}

#[inline(always)]
pub unsafe fn _mm_permutevar_pd_v2(value: __m128d, indices: __m128i) -> __m128d {
    _mm_castsi128_pd(_mm_permutevarx_epi64x_v2(_mm_castpd_si128(value), indices))
}

#[inline(always)]
pub unsafe fn _mm_blendv_epi32x_v2(xmm0: __m128i, xmm1: __m128i, mask: __m128i) -> __m128i {
    _mm_castps_si128(_mm_blendv_ps(
        _mm_castsi128_ps(xmm0),
        _mm_castsi128_ps(xmm1),
        _mm_castsi128_ps(mask),
    ))
}

#[inline(always)]
pub unsafe fn _mm_blendv_epi64x_v2(xmm0: __m128i, xmm1: __m128i, mask: __m128i) -> __m128i {
    _mm_castpd_si128(_mm_blendv_pd(
        _mm_castsi128_pd(xmm0),
        _mm_castsi128_pd(xmm1),
        _mm_castsi128_pd(mask),
    ))
}