fgf 3.0.0

Faster Galois Fields: SIMD and portable finite-field arithmetic (binary towers and prime fields) and vector kernels
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115

//! `dst[i] = a[i] * b[i]` over interleaved GF(2^16) elements.
//!
//! Both operands vary, so no coefficient becomes a table first: the base
//! field multiplies directly with `GF2P8MULB`.

use super::super::{check_elements, swap_mask_avx2};
use crate::kernel::gf16::tail_product;

/// `dst[i] = a[i] * b[i]` over interleaved tower elements using GFNI.
///
/// # Panics
/// Panics unless all three buffers match in length.
#[allow(clippy::used_underscore_binding)]
#[archmage::arcane(import_intrinsics)]
pub fn mul_elementwise_gfni(
    _token: archmage::X64V3GfniCryptoToken,
    dst: &mut [u8],
    a: &[u8],
    b: &[u8],
    b_raw: u8,
    reduction: u8,
) {
    check_elements("gf16::mul_elementwise_gfni", dst.len());
    assert_eq!(
        dst.len(),
        a.len(),
        "gf16::mul_elementwise_gfni: dst is {} bytes but a is {} bytes",
        dst.len(),
        a.len(),
    );
    assert_eq!(
        dst.len(),
        b.len(),
        "gf16::mul_elementwise_gfni: dst is {} bytes but b is {} bytes",
        dst.len(),
        b.len(),
    );
    let swap = swap_mask_avx2();
    let even = _mm256_set1_epi16(0x00ff);
    let delta_even = _mm256_set1_epi16(i16::from_ne_bytes([b_raw, 0]));
    let (a_lanes, a_rest) = a.as_chunks::<32>();
    let (b_lanes, b_rest) = b.as_chunks::<32>();
    let (dst_lanes, dst_rest) = dst.as_chunks_mut::<32>();
    for ((dst_lane, x_lane), y_lane) in dst_lanes.iter_mut().zip(a_lanes).zip(b_lanes) {
        // For x=[a,b], y=[c,d]:
        // constant = ac ^ DELTA*bd
        // extension = ad ^ bc ^ bd.
        let x = _mm256_loadu_si256(x_lane);
        let y = _mm256_loadu_si256(y_lane);
        let direct = _mm256_gf2p8mul_epi8(x, y); // [ac, bd]
        let crossed = _mm256_gf2p8mul_epi8(x, _mm256_shuffle_epi8(y, swap)); // [ad, bc]
        let delta_bd = _mm256_gf2p8mul_epi8(_mm256_shuffle_epi8(direct, swap), delta_even);
        let constant = _mm256_xor_si256(direct, delta_bd);
        let cross_sum = _mm256_xor_si256(crossed, _mm256_shuffle_epi8(crossed, swap));
        let extension = _mm256_xor_si256(cross_sum, direct);
        let product = _mm256_xor_si256(
            _mm256_and_si256(constant, even),
            _mm256_andnot_si256(even, extension),
        );
        _mm256_storeu_si256(dst_lane, product);
    }
    for (d, (x, y)) in dst_rest
        .chunks_exact_mut(2)
        .zip(a_rest.chunks_exact(2).zip(b_rest.chunks_exact(2)))
    {
        d.copy_from_slice(&tail_product([x[0], x[1]], [y[0], y[1]], b_raw, reduction));
    }
}

/// `dst[i] = dst[i] * src[i]` over interleaved tower elements using GFNI.
///
/// # Panics
/// Panics if the slices differ in length.
#[allow(clippy::used_underscore_binding)]
#[archmage::arcane(import_intrinsics)]
pub fn mul_elementwise_assign_gfni(
    _token: archmage::X64V3GfniCryptoToken,
    dst: &mut [u8],
    src: &[u8],
    b_raw: u8,
    reduction: u8,
) {
    check_elements("gf16::mul_elementwise_assign_gfni", dst.len());
    assert_eq!(
        dst.len(),
        src.len(),
        "gf16::mul_elementwise_assign_gfni: dst is {} bytes but src is {} bytes",
        dst.len(),
        src.len(),
    );
    let swap = swap_mask_avx2();
    let even = _mm256_set1_epi16(0x00ff);
    let delta_even = _mm256_set1_epi16(i16::from_ne_bytes([b_raw, 0]));
    let (src_lanes, src_rest) = src.as_chunks::<32>();
    let (dst_lanes, dst_rest) = dst.as_chunks_mut::<32>();
    for (dst_lane, y_lane) in dst_lanes.iter_mut().zip(src_lanes) {
        let x = _mm256_loadu_si256(&*dst_lane);
        let y = _mm256_loadu_si256(y_lane);
        let direct = _mm256_gf2p8mul_epi8(x, y); // [ac, bd]
        let crossed = _mm256_gf2p8mul_epi8(x, _mm256_shuffle_epi8(y, swap)); // [ad, bc]
        let delta_bd = _mm256_gf2p8mul_epi8(_mm256_shuffle_epi8(direct, swap), delta_even);
        let constant = _mm256_xor_si256(direct, delta_bd);
        let cross_sum = _mm256_xor_si256(crossed, _mm256_shuffle_epi8(crossed, swap));
        let extension = _mm256_xor_si256(cross_sum, direct);
        let product = _mm256_xor_si256(
            _mm256_and_si256(constant, even),
            _mm256_andnot_si256(even, extension),
        );
        _mm256_storeu_si256(dst_lane, product);
    }
    for (d, y) in dst_rest.chunks_exact_mut(2).zip(src_rest.chunks_exact(2)) {
        let product = tail_product([d[0], d[1]], [y[0], y[1]], b_raw, reduction);
        d.copy_from_slice(&product);
    }
}