poulpy-cpu-arm 0.7.0

A crate providing concrete NEON accelerated CPU implementations of poulpy-hal through its open extension points
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120

//! Large-coefficient (i128) ring element vector support for [`NTT4x30Neon`](super::NTT4x30Neon).

use super::NTT4x30Neon;
#[cfg(target_arch = "aarch64")]
use poulpy_cpu_ref::reference::ntt4x30::vec_znx_big::AssignOp;
use poulpy_cpu_ref::reference::ntt4x30::{I128BigOps, I128NormalizeOps};

#[cfg(target_arch = "aarch64")]
use crate::neon::normalize::{
    nfc_final_step_assign_neon, nfc_final_step_into_neon, nfc_middle_step_assign_neon, nfc_middle_step_into_neon,
    nfc_middle_step_neon,
};
#[cfg(target_arch = "aarch64")]
use crate::neon::vec_znx_big::{
    vi128_add_assign_neon, vi128_add_neon, vi128_add_small_assign_neon, vi128_add_small_neon, vi128_from_small_neon,
    vi128_neg_from_small_neon, vi128_negate_assign_neon, vi128_negate_neon, vi128_sub_assign_neon, vi128_sub_negate_assign_neon,
    vi128_sub_neon, vi128_sub_small_a_neon, vi128_sub_small_assign_neon, vi128_sub_small_b_neon,
    vi128_sub_small_negate_assign_neon,
};

#[cfg(target_arch = "aarch64")]
impl I128BigOps for NTT4x30Neon {
    #[inline(always)]
    fn i128_add(res: &mut [i128], a: &[i128], b: &[i128]) {
        vi128_add_neon(res.len(), res, a, b);
    }
    #[inline(always)]
    fn i128_add_assign(res: &mut [i128], a: &[i128]) {
        vi128_add_assign_neon(res.len(), res, a);
    }
    #[inline(always)]
    fn i128_add_small(res: &mut [i128], a: &[i128], b: &[i64]) {
        vi128_add_small_neon(res.len(), res, a, b);
    }
    #[inline(always)]
    fn i128_add_small_assign(res: &mut [i128], a: &[i64]) {
        vi128_add_small_assign_neon(res.len(), res, a);
    }
    #[inline(always)]
    fn i128_sub(res: &mut [i128], a: &[i128], b: &[i128]) {
        vi128_sub_neon(res.len(), res, a, b);
    }
    #[inline(always)]
    fn i128_sub_assign(res: &mut [i128], a: &[i128]) {
        vi128_sub_assign_neon(res.len(), res, a);
    }
    #[inline(always)]
    fn i128_sub_negate_assign(res: &mut [i128], a: &[i128]) {
        vi128_sub_negate_assign_neon(res.len(), res, a);
    }
    #[inline(always)]
    fn i128_sub_small_a(res: &mut [i128], a: &[i64], b: &[i128]) {
        vi128_sub_small_a_neon(res.len(), res, a, b);
    }
    #[inline(always)]
    fn i128_sub_small_b(res: &mut [i128], a: &[i128], b: &[i64]) {
        vi128_sub_small_b_neon(res.len(), res, a, b);
    }
    #[inline(always)]
    fn i128_sub_small_assign(res: &mut [i128], a: &[i64]) {
        vi128_sub_small_assign_neon(res.len(), res, a);
    }
    #[inline(always)]
    fn i128_sub_small_negate_assign(res: &mut [i128], a: &[i64]) {
        vi128_sub_small_negate_assign_neon(res.len(), res, a);
    }
    #[inline(always)]
    fn i128_negate(res: &mut [i128], a: &[i128]) {
        vi128_negate_neon(res.len(), res, a);
    }
    #[inline(always)]
    fn i128_negate_assign(res: &mut [i128]) {
        vi128_negate_assign_neon(res.len(), res);
    }
    #[inline(always)]
    fn i128_neg_from_small(res: &mut [i128], a: &[i64]) {
        vi128_neg_from_small_neon(res.len(), res, a);
    }
    #[inline(always)]
    fn i128_from_small(res: &mut [i128], a: &[i64]) {
        vi128_from_small_neon(res.len(), res, a);
    }
}

#[cfg(not(target_arch = "aarch64"))]
impl I128BigOps for NTT4x30Neon {}

impl poulpy_cpu_ref::hal_defaults::ScalarBigHadamardProduct for NTT4x30Neon {
    #[inline(always)]
    fn scalar_big_hadamard_product(res: &mut [i128], a: &[i64], b: &[i64]) {
        <Self as I128BigOps>::i128_hadamard_product_i64(res, a, b)
    }
}

#[cfg(target_arch = "aarch64")]
impl I128NormalizeOps for NTT4x30Neon {
    #[inline(always)]
    fn nfc_middle_step(base2k: usize, lsh: usize, res: &mut [i64], a: &[i128], carry: &mut [i128]) {
        nfc_middle_step_neon(base2k, lsh, res, a, carry);
    }
    #[inline(always)]
    fn nfc_middle_step_into<O: AssignOp>(base2k: usize, lsh: usize, res: &mut [i64], a: &[i128], carry: &mut [i128]) {
        nfc_middle_step_into_neon::<O>(base2k, lsh, res, a, carry);
    }
    #[inline(always)]
    fn nfc_middle_step_assign(base2k: usize, lsh: usize, res: &mut [i64], carry: &mut [i128]) {
        nfc_middle_step_assign_neon(base2k, lsh, res, carry);
    }
    #[inline(always)]
    fn nfc_final_step_assign(base2k: usize, lsh: usize, res: &mut [i64], carry: &mut [i128]) {
        nfc_final_step_assign_neon(base2k, lsh, res, carry);
    }
    #[inline(always)]
    fn nfc_final_step_into<O: AssignOp>(base2k: usize, lsh: usize, res: &mut [i64], carry: &mut [i128]) {
        nfc_final_step_into_neon::<O>(base2k, lsh, res, carry);
    }
}

#[cfg(not(target_arch = "aarch64"))]
impl I128NormalizeOps for NTT4x30Neon {}