conspire 0.7.2

The Rust interface to conspire.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140

use crate::math::Scalar;
use std::arch::x86_64::{
    _mm256_fmadd_pd, _mm256_fnmadd_pd, _mm256_loadu_pd, _mm256_set1_pd, _mm256_storeu_pd,
};

#[target_feature(enable = "avx2", enable = "fma")]
pub(super) unsafe fn axpy(target: &mut [Scalar], column: &[Scalar], w: Scalar) {
    let len = target.len();
    let spread = _mm256_set1_pd(w);
    let mut r = 0;
    unsafe {
        while r + 4 <= len {
            let entry = target.as_mut_ptr().add(r);
            _mm256_storeu_pd(
                entry,
                _mm256_fnmadd_pd(
                    _mm256_loadu_pd(column.as_ptr().add(r)),
                    spread,
                    _mm256_loadu_pd(entry),
                ),
            );
            r += 4;
        }
    }
    (r..len).for_each(|i| target[i] -= column[i] * w);
}

#[target_feature(enable = "avx2", enable = "fma")]
pub(super) unsafe fn rank_one_quad(
    temp_0: &mut [Scalar],
    temp_1: &mut [Scalar],
    temp_2: &mut [Scalar],
    temp_3: &mut [Scalar],
    column: &[Scalar],
    u: [Scalar; 4],
) {
    let len = column.len();
    let u_0 = _mm256_set1_pd(u[0]);
    let u_1 = _mm256_set1_pd(u[1]);
    let u_2 = _mm256_set1_pd(u[2]);
    let u_3 = _mm256_set1_pd(u[3]);
    let mut r = 0;
    unsafe {
        while r + 4 <= len {
            let value = _mm256_loadu_pd(column.as_ptr().add(r));
            _mm256_storeu_pd(
                temp_0.as_mut_ptr().add(r),
                _mm256_fmadd_pd(value, u_0, _mm256_loadu_pd(temp_0.as_ptr().add(r))),
            );
            _mm256_storeu_pd(
                temp_1.as_mut_ptr().add(r),
                _mm256_fmadd_pd(value, u_1, _mm256_loadu_pd(temp_1.as_ptr().add(r))),
            );
            _mm256_storeu_pd(
                temp_2.as_mut_ptr().add(r),
                _mm256_fmadd_pd(value, u_2, _mm256_loadu_pd(temp_2.as_ptr().add(r))),
            );
            _mm256_storeu_pd(
                temp_3.as_mut_ptr().add(r),
                _mm256_fmadd_pd(value, u_3, _mm256_loadu_pd(temp_3.as_ptr().add(r))),
            );
            r += 4;
        }
    }
    (r..len).for_each(|i| {
        temp_0[i] += column[i] * u[0];
        temp_1[i] += column[i] * u[1];
        temp_2[i] += column[i] * u[2];
        temp_3[i] += column[i] * u[3];
    });
}

#[allow(clippy::too_many_arguments)]
#[target_feature(enable = "avx2", enable = "fma")]
pub(super) unsafe fn rank_two_quad(
    temp_0: &mut [Scalar],
    temp_1: &mut [Scalar],
    temp_2: &mut [Scalar],
    temp_3: &mut [Scalar],
    column: &[Scalar],
    other: &[Scalar],
    u: [Scalar; 4],
    w: [Scalar; 4],
) {
    let len = column.len();
    let u_0 = _mm256_set1_pd(u[0]);
    let u_1 = _mm256_set1_pd(u[1]);
    let u_2 = _mm256_set1_pd(u[2]);
    let u_3 = _mm256_set1_pd(u[3]);
    let w_0 = _mm256_set1_pd(w[0]);
    let w_1 = _mm256_set1_pd(w[1]);
    let w_2 = _mm256_set1_pd(w[2]);
    let w_3 = _mm256_set1_pd(w[3]);
    let mut r = 0;
    unsafe {
        while r + 4 <= len {
            let value = _mm256_loadu_pd(column.as_ptr().add(r));
            let second = _mm256_loadu_pd(other.as_ptr().add(r));
            _mm256_storeu_pd(
                temp_0.as_mut_ptr().add(r),
                _mm256_fmadd_pd(
                    second,
                    w_0,
                    _mm256_fmadd_pd(value, u_0, _mm256_loadu_pd(temp_0.as_ptr().add(r))),
                ),
            );
            _mm256_storeu_pd(
                temp_1.as_mut_ptr().add(r),
                _mm256_fmadd_pd(
                    second,
                    w_1,
                    _mm256_fmadd_pd(value, u_1, _mm256_loadu_pd(temp_1.as_ptr().add(r))),
                ),
            );
            _mm256_storeu_pd(
                temp_2.as_mut_ptr().add(r),
                _mm256_fmadd_pd(
                    second,
                    w_2,
                    _mm256_fmadd_pd(value, u_2, _mm256_loadu_pd(temp_2.as_ptr().add(r))),
                ),
            );
            _mm256_storeu_pd(
                temp_3.as_mut_ptr().add(r),
                _mm256_fmadd_pd(
                    second,
                    w_3,
                    _mm256_fmadd_pd(value, u_3, _mm256_loadu_pd(temp_3.as_ptr().add(r))),
                ),
            );
            r += 4;
        }
    }
    (r..len).for_each(|i| {
        temp_0[i] += column[i] * u[0] + other[i] * w[0];
        temp_1[i] += column[i] * u[1] + other[i] * w[1];
        temp_2[i] += column[i] * u[2] + other[i] * w[2];
        temp_3[i] += column[i] * u[3] + other[i] * w[3];
    });
}