1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
/*--------------------------------------------------------------------
* Symbols referenced in this file:
* - pg_popcount64
* - pg_popcount32
*--------------------------------------------------------------------
*/
/*-------------------------------------------------------------------------
*
* pg_popcount_aarch64.c
* Holds the AArch64 popcount implementations.
*
* Copyright (c) 2025, PostgreSQL Global Development Group
*
* IDENTIFICATION
* src/port/pg_popcount_aarch64.c
*
*-------------------------------------------------------------------------
*/
#include "c.h"
#include "port/pg_bitutils.h"
#ifdef POPCNT_AARCH64
#include <arm_neon.h>
#ifdef USE_SVE_POPCNT_WITH_RUNTIME_CHECK
#include <arm_sve.h>
#if defined(HAVE_ELF_AUX_INFO) || defined(HAVE_GETAUXVAL)
#include <sys/auxv.h>
/* Ancient glibc releases don't include the HWCAPxxx macros in sys/auxv.h */
#if defined(__linux__) && !defined(HWCAP_SVE)
#include <asm/hwcap.h>
#endif
#endif
#endif
/*
* The Neon versions are built regardless of whether we are building the SVE
* versions.
*/
static uint64 pg_popcount_neon(const char *buf, int bytes);
static uint64 pg_popcount_masked_neon(const char *buf, int bytes, bits8 mask);
#ifdef USE_SVE_POPCNT_WITH_RUNTIME_CHECK
/*
* These are the SVE implementations of the popcount functions.
*/
static uint64 pg_popcount_sve(const char *buf, int bytes);
static uint64 pg_popcount_masked_sve(const char *buf, int bytes, bits8 mask);
/*
* The function pointers are initially set to "choose" functions. These
* functions will first set the pointers to the right implementations (based on
* what the current CPU supports) and then will call the pointer to fulfill the
* caller's request.
*/
static uint64 pg_popcount_choose(const char *buf, int bytes);
static uint64 pg_popcount_masked_choose(const char *buf, int bytes, bits8 mask);
#ifdef HAVE_ELF_AUX_INFO
#else
#endif
/*
* pg_popcount_sve
* Returns number of 1 bits in buf
*/
/*
* pg_popcount_masked_sve
* Returns number of 1 bits in buf after applying the mask to each byte
*/
#else /* USE_SVE_POPCNT_WITH_RUNTIME_CHECK */
/*
* When the SVE version isn't available, there's no point in using function
* pointers to vary the implementation. We instead just make these actual
* external functions when USE_SVE_POPCNT_WITH_RUNTIME_CHECK is not defined.
* The compiler should be able to inline the Neon versions here.
*/
uint64
pg_popcount_optimized(const char *buf, int bytes)
{
return pg_popcount_neon(buf, bytes);
}
uint64
pg_popcount_masked_optimized(const char *buf, int bytes, bits8 mask)
{
return pg_popcount_masked_neon(buf, bytes, mask);
}
#endif /* ! USE_SVE_POPCNT_WITH_RUNTIME_CHECK */
/*
* pg_popcount32
* Return number of 1 bits in word
*/
int
pg_popcount32(uint32 word)
{
return pg_popcount64((uint64) word);
}
/*
* pg_popcount64
* Return number of 1 bits in word
*/
int
pg_popcount64(uint64 word)
{
/*
* For some compilers, __builtin_popcountl() already emits Neon
* instructions. The line below should compile to the same code on those
* systems.
*/
return vaddv_u8(vcnt_u8(vld1_u8((const uint8 *) &word)));
}
/*
* pg_popcount_neon
* Returns number of 1 bits in buf
*/
/*
* pg_popcount_masked_neon
* Returns number of 1 bits in buf after applying the mask to each byte
*/
#else /* POPCNT_AARCH64 */
/* prevent linker complaints about empty module */
extern int pg_popcount_aarch64_dummy_variable;
int pg_popcount_aarch64_dummy_variable = 0;
#endif /* ! POPCNT_AARCH64 */