hashcrew 0.2.0

Fast, portable hashing for non-cryptographic use
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164

// Copyright 2026 FastLabs Developers
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// These kernels adapt xxHash's SSE2/AVX2 routines and twox-hash 2.1.4's Rust
// stripe accumulation. Scrambling follows xxHash's SIMD implementations.
// Copyright (C) 2012-2023 Yann Collet
// Copyright (c) 2015 Jake Goulding
// Hashcrew's modifications are Apache-2.0; the xxHash and twox-hash portions
// retain their BSD-2-Clause and MIT terms, respectively. See LICENSE.
// Reference sources:
// https://github.com/Cyan4973/xxHash/blob/e626a72bc2321cd320e953a0ccf1584cad60f363/xxhash.h
// https://github.com/shepmaster/twox-hash/blob/6f866bffe73900c63df2650be4eed41e3ed9b500/src/xxhash3/large/sse2.rs
// https://github.com/shepmaster/twox-hash/blob/6f866bffe73900c63df2650be4eed41e3ed9b500/src/xxhash3/large/avx2.rs

use core::arch::x86_64::*;

use super::Xxh3Kernel;

const PRIME32_1: i64 = 0x9e37_79b1;

#[derive(Clone, Copy)]
pub(crate) struct Sse2(());

impl Sse2 {
    pub(crate) unsafe fn new_unchecked() -> Self {
        Self(())
    }
}

impl Xxh3Kernel for Sse2 {
    fn accumulate(self, acc: &mut [u64; 8], stripe: &[u8; 64], secret: &[u8; 64]) {
        // SAFETY: This type is only constructed after SSE2 feature detection.
        unsafe { accumulate_sse2(acc, stripe, secret) }
    }

    fn scramble(self, acc: &mut [u64; 8], secret: &[u8; 64]) {
        // SAFETY: This type is only constructed after SSE2 feature detection.
        unsafe { scramble_sse2(acc, secret) }
    }
}

#[target_feature(enable = "sse2")]
unsafe fn accumulate_sse2(acc: &mut [u64; 8], stripe: &[u8; 64], secret: &[u8; 64]) {
    let acc_ptr = acc.as_mut_ptr().cast::<__m128i>();
    let stripe_ptr = stripe.as_ptr().cast::<__m128i>();
    let secret_ptr = secret.as_ptr().cast::<__m128i>();

    for index in 0..4 {
        // SAFETY: All pointers address two lanes within fixed-size arrays. Unaligned
        // loads and stores are used, and the caller guarantees SSE2 support.
        unsafe {
            let mut acc_vec = _mm_loadu_si128(acc_ptr.add(index));
            let data = _mm_loadu_si128(stripe_ptr.add(index));
            let key = _mm_xor_si128(data, _mm_loadu_si128(secret_ptr.add(index)));
            let swapped = _mm_shuffle_epi32::<0b01_00_11_10>(data);
            let product = _mm_mul_epu32(key, _mm_srli_epi64::<32>(key));
            acc_vec = _mm_add_epi64(acc_vec, swapped);
            acc_vec = _mm_add_epi64(acc_vec, product);
            _mm_storeu_si128(acc_ptr.add(index), acc_vec);
        }
    }
}

#[target_feature(enable = "sse2")]
unsafe fn scramble_sse2(acc: &mut [u64; 8], secret: &[u8; 64]) {
    let acc_ptr = acc.as_mut_ptr().cast::<__m128i>();
    let secret_ptr = secret.as_ptr().cast::<__m128i>();
    // SAFETY: The caller guarantees SSE2 support. This intrinsic is unsafe on
    // the MSRV and safe on newer compilers.
    #[allow(unused_unsafe)]
    let factor = unsafe { _mm_set1_epi64x(PRIME32_1) };

    for index in 0..4 {
        // SAFETY: All pointers address two lanes within fixed-size arrays. Unaligned
        // loads and stores are used, and the caller guarantees SSE2 support.
        unsafe {
            let mut value = _mm_loadu_si128(acc_ptr.add(index));
            value = _mm_xor_si128(value, _mm_srli_epi64::<47>(value));
            value = _mm_xor_si128(value, _mm_loadu_si128(secret_ptr.add(index)));
            let low_product = _mm_mul_epu32(value, factor);
            let high_product = _mm_mul_epu32(_mm_srli_epi64::<32>(value), factor);
            value = _mm_add_epi64(low_product, _mm_slli_epi64::<32>(high_product));
            _mm_storeu_si128(acc_ptr.add(index), value);
        }
    }
}

#[derive(Clone, Copy)]
pub(crate) struct Avx2(());

impl Avx2 {
    pub(crate) unsafe fn new_unchecked() -> Self {
        Self(())
    }
}

impl Xxh3Kernel for Avx2 {
    fn accumulate(self, acc: &mut [u64; 8], stripe: &[u8; 64], secret: &[u8; 64]) {
        // SAFETY: This type is only constructed after AVX2 feature detection.
        unsafe { accumulate_avx2(acc, stripe, secret) }
    }

    fn scramble(self, acc: &mut [u64; 8], secret: &[u8; 64]) {
        // SAFETY: This type is only constructed after AVX2 feature detection.
        unsafe { scramble_avx2(acc, secret) }
    }
}

#[target_feature(enable = "avx2")]
unsafe fn accumulate_avx2(acc: &mut [u64; 8], stripe: &[u8; 64], secret: &[u8; 64]) {
    let acc_ptr = acc.as_mut_ptr().cast::<__m256i>();
    let stripe_ptr = stripe.as_ptr().cast::<__m256i>();
    let secret_ptr = secret.as_ptr().cast::<__m256i>();

    for index in 0..2 {
        // SAFETY: All pointers address four lanes within fixed-size arrays. Unaligned
        // loads and stores are used, and the caller guarantees AVX2 support.
        unsafe {
            let mut acc_vec = _mm256_loadu_si256(acc_ptr.add(index));
            let data = _mm256_loadu_si256(stripe_ptr.add(index));
            let key = _mm256_xor_si256(data, _mm256_loadu_si256(secret_ptr.add(index)));
            let swapped = _mm256_shuffle_epi32::<0b01_00_11_10>(data);
            let product = _mm256_mul_epu32(key, _mm256_srli_epi64::<32>(key));
            acc_vec = _mm256_add_epi64(acc_vec, swapped);
            acc_vec = _mm256_add_epi64(acc_vec, product);
            _mm256_storeu_si256(acc_ptr.add(index), acc_vec);
        }
    }
}

#[target_feature(enable = "avx2")]
unsafe fn scramble_avx2(acc: &mut [u64; 8], secret: &[u8; 64]) {
    let acc_ptr = acc.as_mut_ptr().cast::<__m256i>();
    let secret_ptr = secret.as_ptr().cast::<__m256i>();
    // SAFETY: The caller guarantees AVX2 support. This intrinsic is unsafe on
    // the MSRV and safe on newer compilers.
    #[allow(unused_unsafe)]
    let factor = unsafe { _mm256_set1_epi64x(PRIME32_1) };

    for index in 0..2 {
        // SAFETY: All pointers address four lanes within fixed-size arrays. Unaligned
        // loads and stores are used, and the caller guarantees AVX2 support.
        unsafe {
            let mut value = _mm256_loadu_si256(acc_ptr.add(index));
            value = _mm256_xor_si256(value, _mm256_srli_epi64::<47>(value));
            value = _mm256_xor_si256(value, _mm256_loadu_si256(secret_ptr.add(index)));
            let low_product = _mm256_mul_epu32(value, factor);
            let high_product = _mm256_mul_epu32(_mm256_srli_epi64::<32>(value), factor);
            value = _mm256_add_epi64(low_product, _mm256_slli_epi64::<32>(high_product));
            _mm256_storeu_si256(acc_ptr.add(index), value);
        }
    }
}