crc64fast 1.1.0

SIMD accelerated CRC64 calculation
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93

// Copyright 2020 TiKV Project Authors. Licensed under MIT or Apache-2.0.

//! AArch64 implementation of the PCLMULQDQ-based CRC calculation.

use std::arch::{aarch64::*, is_aarch64_feature_detected};
use std::mem::transmute;
use std::ops::BitXor;

#[repr(transparent)]
#[derive(Copy, Clone, Debug)]
pub struct Simd(uint8x16_t);

#[allow(non_camel_case_types)]
type poly64_t = u64;

impl Simd {
    #[inline]
    #[target_feature(enable = "neon")]
    unsafe fn from_mul(a: poly64_t, b: poly64_t) -> Self {
        let mul = vmull_p64(a, b);
        Self(vreinterpretq_u8_p128(mul))
    }

    #[inline]
    #[target_feature(enable = "neon")]
    unsafe fn into_poly64s(self) -> [poly64_t; 2] {
        let x = vreinterpretq_p64_u8(self.0);
        [vgetq_lane_p64(x, 0), vgetq_lane_p64(x, 1)]
    }

    #[inline]
    #[target_feature(enable = "neon")]
    unsafe fn high_64(self) -> poly64_t {
        let x = vreinterpretq_p64_u8(self.0);
        vgetq_lane_p64(x, 1)
    }

    #[inline]
    #[target_feature(enable = "neon")]
    unsafe fn low_64(self) -> poly64_t {
        let x = vreinterpretq_p64_u8(self.0);
        vgetq_lane_p64(x, 0)
    }
}

impl super::SimdExt for Simd {
    fn is_supported() -> bool {
        is_aarch64_feature_detected!("pmull") && is_aarch64_feature_detected!("neon")
    }

    #[inline]
    #[target_feature(enable = "neon")]
    unsafe fn new(high: u64, low: u64) -> Self {
        Self(vcombine_u8(vcreate_u8(low), vcreate_u8(high)))
    }

    #[inline]
    #[target_feature(enable = "aes", enable = "neon")]
    unsafe fn fold_16(self, coeff: Self) -> Self {
        let [x0, x1] = self.into_poly64s();
        let [c0, c1] = coeff.into_poly64s();
        let h = Self::from_mul(c0, x0);
        let l = Self::from_mul(c1, x1);
        h ^ l
    }

    #[inline]
    #[target_feature(enable = "neon")]
    unsafe fn fold_8(self, coeff: u64) -> Self {
        let [x0, x1] = self.into_poly64s();
        let h = Self::from_mul(transmute(coeff), x0);
        let l = Self::new(0, transmute(x1));
        h ^ l
    }

    #[inline]
    #[target_feature(enable = "neon")]
    unsafe fn barrett(self, poly: u64, mu: u64) -> u64 {
        let t1 = Self::from_mul(self.low_64(), transmute(mu)).low_64();
        let l = Self::from_mul(t1, transmute(poly));
        let reduced: u64 = transmute((self ^ l).high_64());
        let t1: u64 = transmute(t1);
        reduced ^ t1
    }
}

impl BitXor for Simd {
    type Output = Simd;

    fn bitxor(self, other: Self) -> Self {
        unsafe { Self(veorq_u8(self.0, other.0)) }
    }
}