1use core::ops::{Add, AddAssign, BitAnd, BitOr, BitXor, BitXorAssign, Not};
2use crypto_simd::*;
3
4macro_rules! define_vec1 {
5 ($X1:ident, $word:ident) => {
6 #[allow(non_camel_case_types)]
7 #[derive(Copy, Clone)]
8 pub struct $X1($word);
9 impl $X1 {
10 #[inline(always)]
11 pub fn new(a: $word) -> Self {
12 $X1(a)
13 }
14 #[inline(always)]
15 pub fn rotate_right(&mut self, i: $word) {
16 self.0 = self.0.rotate_right(i as u32);
17 }
18 #[inline(always)]
19 pub fn load(xs: &[$word]) -> Self {
20 debug_assert_eq!(xs.len(), 1);
21 $X1(xs[0])
22 }
23 #[inline(always)]
24 pub fn xor_store(self, xs: &mut [$word]) {
25 debug_assert_eq!(xs.len(), 1);
26 xs[0] ^= self.0;
27 }
28 #[inline(always)]
29 pub fn into_inner(self) -> $word {
30 self.0
31 }
32 #[inline(always)]
33 fn swap(self, m: u128, i: u32) -> Self {
34 $X1(((self.0 & m) >> i) | ((self.0) << i) & m)
35 }
36 #[inline(always)]
37 pub fn swap1(self) -> Self {
38 self.swap(0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa, 1)
39 }
40 #[inline(always)]
41 pub fn swap2(self) -> Self {
42 self.swap(0xcccccccccccccccccccccccccccccccc, 2)
43 }
44 #[inline(always)]
45 pub fn swap4(self) -> Self {
46 self.swap(0xf0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0, 4)
47 }
48 #[inline(always)]
49 pub fn swap8(self) -> Self {
50 self.swap(0xff00ff00ff00ff00ff00ff00ff00ff00, 8)
51 }
52 #[inline(always)]
53 pub fn swap16(self) -> Self {
54 self.swap(0xffff0000ffff0000ffff0000ffff0000, 16)
55 }
56 #[inline(always)]
57 pub fn swap32(self) -> Self {
58 self.swap(0xffffffff00000000ffffffff00000000, 32)
59 }
60 #[inline(always)]
61 pub fn swap64(self) -> Self {
62 $X1(self.0 << 64 | self.0 >> 64)
63 }
64 #[inline(always)]
65 pub fn andnot(self, rhs: Self) -> Self {
66 !self & rhs
67 }
68 #[inline(always)]
69 pub fn extract(self, i: u32) -> $word {
70 debug_assert_eq!(i, 0);
71 self.0
72 }
73 }
74 impl AddAssign for $X1 {
75 #[inline(always)]
76 fn add_assign(&mut self, rhs: Self) {
77 self.0 += rhs.0;
78 }
79 }
80 impl BitXorAssign for $X1 {
81 #[inline(always)]
82 fn bitxor_assign(&mut self, rhs: Self) {
83 self.0 ^= rhs.0;
84 }
85 }
86 impl BitXor for $X1 {
87 type Output = Self;
88 #[inline(always)]
89 fn bitxor(self, rhs: Self) -> Self::Output {
90 $X1(self.0 ^ rhs.0)
91 }
92 }
93 impl BitAnd for $X1 {
94 type Output = Self;
95 #[inline(always)]
96 fn bitand(self, rhs: Self) -> Self::Output {
97 $X1(self.0 & rhs.0)
98 }
99 }
100 impl Not for $X1 {
101 type Output = Self;
102 #[inline(always)]
103 fn not(self) -> Self::Output {
104 $X1(!self.0)
105 }
106 }
107 };
108}
109macro_rules! define_vec2 {
110 ($X2:ident, $word:ident) => {
111 #[allow(non_camel_case_types)]
112 #[derive(Copy, Clone)]
113 pub struct $X2($word, $word);
114 impl $X2 {
115 #[inline(always)]
116 pub fn new(a: $word, b: $word) -> Self {
117 $X2(a, b)
118 }
119 #[inline(always)]
120 fn map<F>(self, mut f: F) -> Self
121 where
122 F: FnMut($word) -> $word,
123 {
124 $X2(f(self.0), f(self.1))
125 }
126 #[inline(always)]
127 fn zipmap<F>(self, rhs: Self, mut f: F) -> Self
128 where
129 F: FnMut($word, $word) -> $word,
130 {
131 $X2(f(self.0, rhs.0), f(self.1, rhs.1))
132 }
133 #[inline(always)]
134 pub fn rotate_right(&mut self, i: $word) {
135 *self = self.map(|x| $word::rotate_right(x, i as u32));
136 }
137 #[inline(always)]
138 pub fn load(xs: &[$word]) -> Self {
139 debug_assert_eq!(xs.len(), 2);
140 $X2(xs[0], xs[1])
141 }
142 #[inline(always)]
143 pub fn xor_store(self, xs: &mut [$word]) {
144 debug_assert_eq!(xs.len(), 2);
145 xs[0] ^= self.0;
146 xs[1] ^= self.1;
147 }
148 #[inline(always)]
149 pub fn extract(self, i: u32) -> $word {
150 let x = [self.0, self.1];
151 x[i as usize]
152 }
153 #[inline(always)]
154 pub fn andnot(self, rhs: Self) -> Self {
155 !self & rhs
156 }
157 }
158 impl AddAssign for $X2 {
159 #[inline(always)]
160 fn add_assign(&mut self, rhs: Self) {
161 *self = self.zipmap(rhs, $word::wrapping_add);
162 }
163 }
164 impl BitXorAssign for $X2 {
165 #[inline(always)]
166 fn bitxor_assign(&mut self, rhs: Self) {
167 *self = self.zipmap(rhs, $word::bitxor);
168 }
169 }
170 impl BitAnd for $X2 {
171 type Output = Self;
172 #[inline(always)]
173 fn bitand(self, rhs: Self) -> Self::Output {
174 $X2(self.0 & rhs.0, self.1 & rhs.1)
175 }
176 }
177 impl Not for $X2 {
178 type Output = Self;
179 #[inline(always)]
180 fn not(self) -> Self::Output {
181 $X2(!self.0, !self.1)
182 }
183 }
184 impl BitOr for $X2 {
185 type Output = Self;
186 #[inline(always)]
187 fn bitor(self, rhs: Self) -> Self::Output {
188 $X2(self.0 | rhs.0, self.1 | rhs.1)
189 }
190 }
191 };
192}
193
194macro_rules! zipmap_impl {
195 ($vec:ident, $word:ident, $trait:ident, $fn:ident, $impl_fn:ident) => {
196 impl $trait for $vec {
197 type Output = Self;
198 #[inline(always)]
199 fn $fn(self, rhs: Self) -> Self::Output {
200 self.zipmap(rhs, $word::$impl_fn)
201 }
202 }
203 };
204 ($vec:ident, $word:ident, $trait:ident, $fn:ident) => {
205 zipmap_impl!($vec, $word, $trait, $fn, $fn);
206 };
207}
208
209macro_rules! define_vec4 {
210 ($X4:ident, $word:ident) => {
211 #[allow(non_camel_case_types)]
212 #[derive(Copy, Clone)]
213 pub struct $X4($word, $word, $word, $word);
214 impl $X4 {
215 #[inline(always)]
216 pub fn new(a: $word, b: $word, c: $word, d: $word) -> Self {
217 $X4(a, b, c, d)
218 }
219 #[inline(always)]
220 fn zipmap<F>(self, rhs: Self, mut f: F) -> Self
221 where
222 F: FnMut($word, $word) -> $word,
223 {
224 $X4(
225 f(self.0, rhs.0),
226 f(self.1, rhs.1),
227 f(self.2, rhs.2),
228 f(self.3, rhs.3),
229 )
230 }
231 #[inline(always)]
232 pub fn rotate_right(&mut self, ii: Self) -> Self {
233 $X4(
234 self.0.rotate_right(ii.0 as u32),
235 self.1.rotate_right(ii.1 as u32),
236 self.2.rotate_right(ii.2 as u32),
237 self.3.rotate_right(ii.3 as u32),
238 )
239 }
240 #[inline(always)]
241 pub fn from_slice_unaligned(xs: &[$word]) -> Self {
242 debug_assert_eq!(xs.len(), 4);
243 $X4(xs[0], xs[1], xs[2], xs[3])
244 }
245 #[inline(always)]
246 pub fn splat(x: $word) -> Self {
247 $X4(x, x, x, x)
248 }
249 #[inline(always)]
250 pub fn write_to_slice_unaligned(self, xs: &mut [$word]) {
251 debug_assert_eq!(xs.len(), 4);
252 xs[0] = self.0;
253 xs[1] = self.1;
254 xs[2] = self.2;
255 xs[3] = self.3;
256 }
257 #[inline(always)]
258 pub fn replace(mut self, i: usize, v: $word) -> Self {
259 let xs = [&mut self.0, &mut self.1, &mut self.2, &mut self.3];
260 *xs[i] = v;
261 self
262 }
263 #[inline(always)]
264 pub fn extract(self, i: usize) -> $word {
265 let xs = [self.0, self.1, self.2, self.3];
266 xs[i]
267 }
268 }
269 impl AddAssign for $X4 {
270 #[inline(always)]
271 fn add_assign(&mut self, rhs: Self) {
272 *self = self.zipmap(rhs, $word::wrapping_add);
273 }
274 }
275 impl BitXorAssign for $X4 {
276 #[inline(always)]
277 fn bitxor_assign(&mut self, rhs: Self) {
278 *self = self.zipmap(rhs, $word::bitxor);
279 }
280 }
281 zipmap_impl!($X4, $word, Add, add, wrapping_add);
282 zipmap_impl!($X4, $word, BitXor, bitxor);
283 zipmap_impl!($X4, $word, BitOr, bitor);
284 zipmap_impl!($X4, $word, BitAnd, bitand);
285 impl RotateWordsRight for $X4 {
286 type Output = Self;
287 #[inline(always)]
288 fn rotate_words_right(self, i: u32) -> Self::Output {
289 debug_assert_eq!(i & !3, 0);
290 match i & 3 {
291 0 => self,
292 1 => $X4(self.3, self.0, self.1, self.2),
293 2 => $X4(self.2, self.3, self.0, self.1),
294 3 => $X4(self.1, self.2, self.3, self.0),
295 _ => unreachable!(),
296 }
297 }
298 }
299 impl SplatRotateRight for $X4 {
300 type Output = Self;
301 #[inline(always)]
302 fn splat_rotate_right(self, i: u32) -> Self::Output {
303 const BITS: u32 = core::mem::size_of::<$word>() as u32 * 8;
304 $X4(
305 (self.0 >> i) | (self.0 << (BITS - i)),
306 (self.1 >> i) | (self.1 << (BITS - i)),
307 (self.2 >> i) | (self.2 << (BITS - i)),
308 (self.3 >> i) | (self.3 << (BITS - i)),
309 )
310 }
311 }
312 };
313}
314
315define_vec4!(u32x4, u32);
316define_vec4!(u64x4, u64);
317define_vec1!(u128x1, u128);
318define_vec2!(u128x2, u128);
319
320#[allow(non_camel_case_types)]
322#[derive(Copy, Clone)]
323pub struct u32x4x4(u32x4, u32x4, u32x4, u32x4);
324impl u32x4x4 {
325 #[inline(always)]
326 fn zipmap<F>(self, rhs: Self, mut f: F) -> Self
327 where
328 F: FnMut(u32x4, u32x4) -> u32x4,
329 {
330 u32x4x4(
331 f(self.0, rhs.0),
332 f(self.1, rhs.1),
333 f(self.2, rhs.2),
334 f(self.3, rhs.3),
335 )
336 }
337 #[inline(always)]
338 pub fn from((a, b, c, d): (u32x4, u32x4, u32x4, u32x4)) -> Self {
339 u32x4x4(a, b, c, d)
340 }
341 #[inline(always)]
342 pub fn splat(a: u32x4) -> Self {
343 u32x4x4(a, a, a, a)
344 }
345 #[inline(always)]
346 pub fn into_parts(self) -> (u32x4, u32x4, u32x4, u32x4) {
347 (self.0, self.1, self.2, self.3)
348 }
349}
350zipmap_impl!(u32x4x4, u32x4, BitXor, bitxor);
351zipmap_impl!(u32x4x4, u32x4, BitOr, bitor);
352zipmap_impl!(u32x4x4, u32x4, BitAnd, bitand);
353zipmap_impl!(u32x4x4, u32x4, Add, add);
354impl BitXorAssign for u32x4x4 {
355 #[inline(always)]
356 fn bitxor_assign(&mut self, rhs: Self) {
357 self.0 = self.0 ^ rhs.0;
358 self.1 = self.1 ^ rhs.1;
359 self.2 = self.2 ^ rhs.2;
360 self.3 = self.3 ^ rhs.3;
361 }
362}
363impl AddAssign for u32x4x4 {
364 #[inline(always)]
365 fn add_assign(&mut self, rhs: Self) {
366 self.0 = self.0 + rhs.0;
367 self.1 = self.1 + rhs.1;
368 self.2 = self.2 + rhs.2;
369 self.3 = self.3 + rhs.3;
370 }
371}
372impl RotateWordsRight for u32x4x4 {
373 type Output = Self;
374 #[inline(always)]
375 fn rotate_words_right(self, i: u32) -> Self::Output {
376 u32x4x4(
377 self.0.rotate_words_right(i),
378 self.1.rotate_words_right(i),
379 self.2.rotate_words_right(i),
380 self.3.rotate_words_right(i),
381 )
382 }
383}
384impl SplatRotateRight for u32x4x4 {
385 type Output = Self;
386 #[inline(always)]
387 fn splat_rotate_right(self, i: u32) -> Self::Output {
388 u32x4x4(
389 self.0.splat_rotate_right(i),
390 self.1.splat_rotate_right(i),
391 self.2.splat_rotate_right(i),
392 self.3.splat_rotate_right(i),
393 )
394 }
395}