#include "sve/sve.hpp"
#include <immintrin.h>
namespace sve::avx512 {
namespace {
inline __m512i load_i32x16(const std::int32_t* p, bool aligned) noexcept {
return aligned ? _mm512_load_si512(p) : _mm512_loadu_si512(p);
}
inline __m512 load_f32x16(const float* p, bool aligned) noexcept {
return aligned ? _mm512_load_ps(p) : _mm512_loadu_ps(p); }
inline unsigned popcnt16(__mmask16 m) noexcept {
#if defined(_MSC_VER)
return __popcnt16(static_cast<unsigned short>(m));
#else
return static_cast<unsigned>(__builtin_popcount(static_cast<unsigned>(m)));
#endif
}
}
std::size_t count_gt_i32(std::span<const std::int32_t> d, std::int32_t thr) {
const auto* p = d.data();
const std::size_t n = d.size();
const bool a = is_aligned(p, 64);
const __m512i vthr = _mm512_set1_epi32(thr); std::uint64_t c0 = 0, c1 = 0, c2 = 0, c3 = 0;
std::size_t i = 0, block = n & ~std::size_t{63};
for (; i < block; i += 64) {
__m512i v0 = load_i32x16(p + i, a);
__m512i v1 = load_i32x16(p + i + 16, a);
__m512i v2 = load_i32x16(p + i + 32, a);
__m512i v3 = load_i32x16(p + i + 48, a);
__mmask16 m0 = _mm512_cmpgt_epi32_mask(v0, vthr);
__mmask16 m1 = _mm512_cmpgt_epi32_mask(v1, vthr);
__mmask16 m2 = _mm512_cmpgt_epi32_mask(v2, vthr);
__mmask16 m3 = _mm512_cmpgt_epi32_mask(v3, vthr);
c0 += popcnt16(m0);
c1 += popcnt16(m1);
c2 += popcnt16(m2);
c3 += popcnt16(m3);
}
auto total = static_cast<std::size_t>(c0 + c1 + c2 + c3);
for (; i < n; ++i) total += static_cast<std::size_t>(p[i] > thr);
return total;
}
std::size_t count_gt_f32(std::span<const float> d, float thr) {
const auto* p = d.data();
const std::size_t n = d.size();
const bool a = is_aligned(p, 64);
const __m512 vthr = _mm512_set1_ps(thr); std::uint64_t c0 = 0, c1 = 0, c2 = 0, c3 = 0;
std::size_t i = 0, block = n & ~std::size_t{63};
for (; i < block; i += 64) {
__m512 v0 = load_f32x16(p + i, a);
__m512 v1 = load_f32x16(p + i + 16, a);
__m512 v2 = load_f32x16(p + i + 32, a);
__m512 v3 = load_f32x16(p + i + 48, a);
c0 += popcnt16(_mm512_cmp_ps_mask(v0, vthr, _CMP_GT_OQ));
c1 += popcnt16(_mm512_cmp_ps_mask(v1, vthr, _CMP_GT_OQ));
c2 += popcnt16(_mm512_cmp_ps_mask(v2, vthr, _CMP_GT_OQ));
c3 += popcnt16(_mm512_cmp_ps_mask(v3, vthr, _CMP_GT_OQ));
}
auto total = static_cast<std::size_t>(c0 + c1 + c2 + c3);
for (; i < n; ++i) total += static_cast<std::size_t>(p[i] > thr);
return total;
}
std::int64_t sum_i32(std::span<const std::int32_t> d) {
const auto* p = d.data();
const std::size_t n = d.size();
const bool a = is_aligned(p, 64);
__m512i a0 = _mm512_setzero_si512(), a1 = _mm512_setzero_si512();
__m512i a2 = _mm512_setzero_si512(), a3 = _mm512_setzero_si512();
std::size_t i = 0, block = n & ~std::size_t{31};
for (; i < block; i += 32) {
__m512i v0 = load_i32x16(p + i, a);
__m512i v1 = load_i32x16(p + i + 16, a);
__m512i lo0 = _mm512_cvtepi32_epi64(_mm512_castsi512_si256(v0));
__m512i hi0 = _mm512_cvtepi32_epi64(_mm512_extracti64x4_epi64(v0, 1));
__m512i lo1 = _mm512_cvtepi32_epi64(_mm512_castsi512_si256(v1));
__m512i hi1 = _mm512_cvtepi32_epi64(_mm512_extracti64x4_epi64(v1, 1));
a0 = _mm512_add_epi64(a0, lo0); a1 = _mm512_add_epi64(a1, hi0);
a2 = _mm512_add_epi64(a2, lo1);
a3 = _mm512_add_epi64(a3, hi1);
}
__m512i acc = _mm512_add_epi64(_mm512_add_epi64(a0, a1), _mm512_add_epi64(a2, a3));
std::int64_t total = _mm512_reduce_add_epi64(acc);
for (; i < n; ++i) total += p[i];
return total;
}
std::optional<size_t> find_i32(std::span<const std::int32_t> d, std::int32_t needle) {
const auto* p = d.data();
const std::size_t n = d.size();
const bool a = is_aligned(p, 64);
const __m512i key = _mm512_set1_epi32(needle);
std::size_t i = 0, block = n & ~std::size_t{63};
for (; i < block; i += 64) {
std::uint64_t bits =
static_cast<std::uint64_t>(_mm512_cmpeq_epi32_mask(load_i32x16(p + i, a), key)) |
(static_cast<std::uint64_t>(_mm512_cmpeq_epi32_mask(load_i32x16(p + i + 16, a), key)) << 16) |
(static_cast<std::uint64_t>(_mm512_cmpeq_epi32_mask(load_i32x16(p + i + 32, a), key)) << 32) |
(static_cast<std::uint64_t>(_mm512_cmpeq_epi32_mask(load_i32x16(p + i + 48, a), key)) << 48);
if (bits) {
unsigned long tz = 0;
#if defined(_MSC_VER)
_BitScanForward64(&tz, bits);
#else
tz = static_cast<unsigned long>(__builtin_ctzll(bits));
#endif
return i + tz;
}
}
for (; i < n; ++i)
if (p[i] == needle) return i;
return std::nullopt;
}
std::size_t filter_into_i32(std::span<const std::int32_t> d, std::int32_t thr,
std::span<std::int32_t> out) {
const auto* p = d.data();
auto* o = out.data();
const std::size_t n = d.size();
const bool a = is_aligned(p, 64);
const __m512i vthr = _mm512_set1_epi32(thr);
std::size_t w = 0, i = 0, block = n & ~std::size_t{15};
for (; i < block; i += 16) {
__m512i v = load_i32x16(p + i, a);
__mmask16 m = _mm512_cmpgt_epi32_mask(v, vthr);
_mm512_mask_compressstoreu_epi32(o + w, m, v);
w += popcnt16(m);
}
for (; i < n; ++i)
if (p[i] > thr) o[w++] = p[i];
return w;
}
}