Skip to main content

simdutf8/implementation/aarch64/
neon.rs

1//! Contains the aarch64 UTF-8 validation implementation.
2
3use core::arch::aarch64::{
4    uint8x16_t, vandq_u8, vdupq_n_u8, veorq_u8, vextq_u8, vld1q_u8, vmaxvq_u8, vmovq_n_u8,
5    vorrq_u8, vqsubq_u8, vqtbl1q_u8, vshrq_n_u8,
6};
7
8use crate::implementation::helpers::Utf8CheckAlgorithm;
9
10// aarch64 SIMD primitives
11
12type SimdU8Value = crate::implementation::helpers::SimdU8Value<uint8x16_t>;
13
14impl SimdU8Value {
15    #[inline]
16    #[allow(clippy::too_many_arguments)]
17    unsafe fn from_32_cut_off_leading(
18        _v0: u8,
19        _v1: u8,
20        _v2: u8,
21        _v3: u8,
22        _v4: u8,
23        _v5: u8,
24        _v6: u8,
25        _v7: u8,
26        _v8: u8,
27        _v9: u8,
28        _v10: u8,
29        _v11: u8,
30        _v12: u8,
31        _v13: u8,
32        _v14: u8,
33        _v15: u8,
34        v16: u8,
35        v17: u8,
36        v18: u8,
37        v19: u8,
38        v20: u8,
39        v21: u8,
40        v22: u8,
41        v23: u8,
42        v24: u8,
43        v25: u8,
44        v26: u8,
45        v27: u8,
46        v28: u8,
47        v29: u8,
48        v30: u8,
49        v31: u8,
50    ) -> Self {
51        let arr: [u8; 16] = [
52            v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31,
53        ];
54        Self::from(vld1q_u8(arr.as_ptr()))
55    }
56
57    #[inline]
58    #[allow(clippy::too_many_arguments)]
59    unsafe fn repeat_16(
60        v0: u8,
61        v1: u8,
62        v2: u8,
63        v3: u8,
64        v4: u8,
65        v5: u8,
66        v6: u8,
67        v7: u8,
68        v8: u8,
69        v9: u8,
70        v10: u8,
71        v11: u8,
72        v12: u8,
73        v13: u8,
74        v14: u8,
75        v15: u8,
76    ) -> Self {
77        let arr: [u8; 16] = [
78            v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15,
79        ];
80        Self::from(vld1q_u8(arr.as_ptr()))
81    }
82
83    #[inline]
84    unsafe fn load_from(ptr: *const u8) -> Self {
85        Self::from(vld1q_u8(ptr))
86    }
87
88    #[inline]
89    unsafe fn lookup_16(self, tbl: Self) -> Self {
90        Self::from(vqtbl1q_u8(tbl.0, self.0))
91    }
92
93    #[inline]
94    unsafe fn splat(val: u8) -> Self {
95        Self::from(vmovq_n_u8(val))
96    }
97
98    #[inline]
99    unsafe fn splat0() -> Self {
100        Self::from(vdupq_n_u8(0))
101    }
102
103    #[inline]
104    unsafe fn or(self, b: Self) -> Self {
105        Self::from(vorrq_u8(self.0, b.0))
106    }
107
108    #[inline]
109    unsafe fn and(self, b: Self) -> Self {
110        Self::from(vandq_u8(self.0, b.0))
111    }
112
113    #[inline]
114    unsafe fn xor(self, b: Self) -> Self {
115        Self::from(veorq_u8(self.0, b.0))
116    }
117
118    #[inline]
119    unsafe fn saturating_sub(self, b: Self) -> Self {
120        Self::from(vqsubq_u8(self.0, b.0))
121    }
122
123    // ugly but shr<N> requires const generics
124
125    #[inline]
126    unsafe fn shr4(self) -> Self {
127        Self::from(vshrq_n_u8(self.0, 4))
128    }
129
130    // ugly but prev<N> requires const generics
131
132    #[inline]
133    unsafe fn prev1(self, prev: Self) -> Self {
134        Self::from(vextq_u8(prev.0, self.0, 16 - 1))
135    }
136
137    // ugly but prev<N> requires const generics
138
139    #[inline]
140    unsafe fn prev2(self, prev: Self) -> Self {
141        Self::from(vextq_u8(prev.0, self.0, 16 - 2))
142    }
143
144    // ugly but prev<N> requires const generics
145
146    #[inline]
147    unsafe fn prev3(self, prev: Self) -> Self {
148        Self::from(vextq_u8(prev.0, self.0, 16 - 3))
149    }
150
151    #[inline]
152    unsafe fn any_bit_set(self) -> bool {
153        vmaxvq_u8(self.0) != 0
154    }
155
156    #[inline]
157    unsafe fn is_ascii(self) -> bool {
158        vmaxvq_u8(self.0) < 0b1000_0000_u8
159    }
160}
161
162impl From<uint8x16_t> for SimdU8Value {
163    #[inline]
164    fn from(val: uint8x16_t) -> Self {
165        Self(val)
166    }
167}
168
169#[inline]
170#[cfg(feature = "aarch64_neon_prefetch")]
171unsafe fn simd_prefetch(ptr: *const u8) {
172    use core::arch::aarch64::{_prefetch, _PREFETCH_LOCALITY3, _PREFETCH_READ};
173    _prefetch(ptr.cast::<i8>(), _PREFETCH_READ, _PREFETCH_LOCALITY3);
174}
175
176#[inline]
177#[cfg(not(feature = "aarch64_neon_prefetch"))]
178unsafe fn simd_prefetch(_ptr: *const u8) {}
179
180#[cfg(feature = "aarch64_neon_prefetch")]
181const PREFETCH: bool = true;
182#[cfg(not(feature = "aarch64_neon_prefetch"))]
183const PREFETCH: bool = false;
184
185use crate::implementation::helpers::TempSimdChunkA16 as TempSimdChunk;
186simd_input_128_bit!();
187algorithm_simd!();
188algorithm_simd_default_special_case_fns!();