#include "sve/sve.hpp"
#include <immintrin.h>
namespace sve::avx2 {
namespace {
inline unsigned popcnt_u32(int mask) noexcept {
#if defined(_MSC_VER)
return __popcnt(static_cast<unsigned>(mask));
#else
return static_cast<unsigned>(__builtin_popcount(static_cast<unsigned>(mask)));
#endif
}
inline std::int32_t hsum_epi32(__m256i v) noexcept {
__m128i hi = _mm256_extracti128_si256(v, 1); __m128i lo = _mm256_castsi256_si128(v); __m128i s = _mm_add_epi32(lo, hi); s = _mm_add_epi32(s, _mm_shuffle_epi32(s, _MM_SHUFFLE(1, 0, 3, 2))); s = _mm_add_epi32(s, _mm_shuffle_epi32(s, _MM_SHUFFLE(2, 3, 0, 1))); return _mm_cvtsi128_si32(s); }
inline std::int64_t hsum_epi64(__m256i v) noexcept {
alignas(32) std::int64_t t[4];
_mm256_store_si256(reinterpret_cast<__m256i*>(t), v); return t[0] + t[1] + t[2] + t[3];
}
inline __m256i load_i32x8(const std::int32_t* p, bool aligned) noexcept {
return aligned ? _mm256_load_si256(reinterpret_cast<const __m256i*>(p)) : _mm256_loadu_si256(reinterpret_cast<const __m256i*>(p)); }
inline __m256 load_f32x8(const float* p, bool aligned) noexcept {
return aligned ? _mm256_load_ps(p) : _mm256_loadu_ps(p); }
struct CompactLut {
alignas(32) std::uint32_t idx[256][8];
};
consteval CompactLut build_lut() {
CompactLut l{};
for (int mask = 0; mask < 256; ++mask) {
int w = 0;
for (int lane = 0; lane < 8; ++lane)
if (mask & (1 << lane)) l.idx[mask][w++] = static_cast<std::uint32_t>(lane);
while (w < 8) l.idx[mask][w++] = 0;
}
return l;
}
constexpr CompactLut kLut = build_lut();
}
std::size_t count_gt_i32(std::span<const std::int32_t> d, std::int32_t thr) {
const auto* p = d.data();
const std::size_t n = d.size();
const bool a = is_aligned(p, 32);
const __m256i vthr = _mm256_set1_epi32(thr); __m256i a0 = _mm256_setzero_si256(), a1 = _mm256_setzero_si256();
__m256i a2 = _mm256_setzero_si256(), a3 = _mm256_setzero_si256();
std::size_t i = 0, block = n & ~std::size_t{31};
for (; i < block; i += 32) {
__m256i v0 = load_i32x8(p + i, a);
__m256i v1 = load_i32x8(p + i + 8, a);
__m256i v2 = load_i32x8(p + i + 16, a);
__m256i v3 = load_i32x8(p + i + 24, a);
__m256i m0 = _mm256_cmpgt_epi32(v0, vthr);
__m256i m1 = _mm256_cmpgt_epi32(v1, vthr);
__m256i m2 = _mm256_cmpgt_epi32(v2, vthr);
__m256i m3 = _mm256_cmpgt_epi32(v3, vthr);
a0 = _mm256_sub_epi32(a0, m0);
a1 = _mm256_sub_epi32(a1, m1);
a2 = _mm256_sub_epi32(a2, m2);
a3 = _mm256_sub_epi32(a3, m3);
}
__m256i acc = _mm256_add_epi32(_mm256_add_epi32(a0, a1), _mm256_add_epi32(a2, a3));
auto total = static_cast<std::size_t>(hsum_epi32(acc));
for (; i < n; ++i) total += static_cast<std::size_t>(p[i] > thr); return total;
}
std::size_t count_gt_f32(std::span<const float> d, float thr) {
const auto* p = d.data();
const std::size_t n = d.size();
const bool a = is_aligned(p, 32);
const __m256 vthr = _mm256_set1_ps(thr); unsigned c0 = 0, c1 = 0, c2 = 0, c3 = 0;
std::size_t i = 0, block = n & ~std::size_t{31};
for (; i < block; i += 32) {
__m256 v0 = load_f32x8(p + i, a);
__m256 v1 = load_f32x8(p + i + 8, a);
__m256 v2 = load_f32x8(p + i + 16, a);
__m256 v3 = load_f32x8(p + i + 24, a);
__m256 m0 = _mm256_cmp_ps(v0, vthr, _CMP_GT_OQ);
__m256 m1 = _mm256_cmp_ps(v1, vthr, _CMP_GT_OQ);
__m256 m2 = _mm256_cmp_ps(v2, vthr, _CMP_GT_OQ);
__m256 m3 = _mm256_cmp_ps(v3, vthr, _CMP_GT_OQ);
c0 += static_cast<unsigned>(popcnt_u32(_mm256_movemask_ps(m0)));
c1 += static_cast<unsigned>(popcnt_u32(_mm256_movemask_ps(m1)));
c2 += static_cast<unsigned>(popcnt_u32(_mm256_movemask_ps(m2)));
c3 += static_cast<unsigned>(popcnt_u32(_mm256_movemask_ps(m3)));
}
std::size_t total = c0 + c1 + c2 + c3;
for (; i < n; ++i) total += static_cast<std::size_t>(p[i] > thr);
return total;
}
std::int64_t sum_i32(std::span<const std::int32_t> d) {
const auto* p = d.data();
const std::size_t n = d.size();
const bool a = is_aligned(p, 32);
__m256i a0 = _mm256_setzero_si256(), a1 = _mm256_setzero_si256();
__m256i a2 = _mm256_setzero_si256(), a3 = _mm256_setzero_si256();
std::size_t i = 0, block = n & ~std::size_t{15};
for (; i < block; i += 16) {
__m256i v0 = load_i32x8(p + i, a);
__m256i v1 = load_i32x8(p + i + 8, a);
__m256i lo0 = _mm256_cvtepi32_epi64(_mm256_castsi256_si128(v0));
__m256i hi0 = _mm256_cvtepi32_epi64(_mm256_extracti128_si256(v0, 1));
__m256i lo1 = _mm256_cvtepi32_epi64(_mm256_castsi256_si128(v1));
__m256i hi1 = _mm256_cvtepi32_epi64(_mm256_extracti128_si256(v1, 1));
a0 = _mm256_add_epi64(a0, lo0);
a1 = _mm256_add_epi64(a1, hi0);
a2 = _mm256_add_epi64(a2, lo1);
a3 = _mm256_add_epi64(a3, hi1);
}
__m256i acc = _mm256_add_epi64(_mm256_add_epi64(a0, a1), _mm256_add_epi64(a2, a3));
std::int64_t total = hsum_epi64(acc);
for (; i < n; ++i) total += p[i];
return total;
}
std::optional<size_t> find_i32(std::span<const std::int32_t> d, std::int32_t needle) {
const auto* p = d.data();
const std::size_t n = d.size();
const bool a = is_aligned(p, 32);
const __m256i key = _mm256_set1_epi32(needle);
std::size_t i = 0, block = n & ~std::size_t{31};
for (; i < block; i += 32) {
__m256i v0 = load_i32x8(p + i, a);
__m256i v1 = load_i32x8(p + i + 8, a);
__m256i v2 = load_i32x8(p + i + 16, a);
__m256i v3 = load_i32x8(p + i + 24, a);
__m256i m0 = _mm256_cmpeq_epi32(v0, key);
__m256i m1 = _mm256_cmpeq_epi32(v1, key);
__m256i m2 = _mm256_cmpeq_epi32(v2, key);
__m256i m3 = _mm256_cmpeq_epi32(v3, key);
__m256i any = _mm256_or_si256(_mm256_or_si256(m0, m1), _mm256_or_si256(m2, m3));
if (!_mm256_testz_si256(any, any)) {
const __m256i ms[4] = {m0, m1, m2, m3};
for (int k = 0; k < 4; ++k) {
int bits = _mm256_movemask_ps(_mm256_castsi256_ps(ms[k]));
if (bits) {
unsigned long tz = 0;
#if defined(_MSC_VER)
_BitScanForward(&tz, static_cast<unsigned long>(bits));
#else
tz = static_cast<unsigned long>(__builtin_ctz(static_cast<unsigned>(bits)));
#endif
return i + static_cast<std::size_t>(k) * 8 + tz;
}
}
}
}
for (; i < n; ++i)
if (p[i] == needle) return i;
return std::nullopt;
}
std::size_t filter_into_i32(std::span<const std::int32_t> d, std::int32_t thr,
std::span<std::int32_t> out) {
const auto* p = d.data();
auto* o = out.data();
const std::size_t n = d.size();
const bool a = is_aligned(p, 32);
const __m256i vthr = _mm256_set1_epi32(thr);
std::size_t w = 0, i = 0, block = n & ~std::size_t{7};
for (; i < block; i += 8) {
__m256i v = load_i32x8(p + i, a);
__m256i m = _mm256_cmpgt_epi32(v, vthr);
int bits = _mm256_movemask_ps(_mm256_castsi256_ps(m));
if (bits) {
__m256i idx = _mm256_load_si256(reinterpret_cast<const __m256i*>(kLut.idx[bits]));
__m256i packed = _mm256_permutevar8x32_epi32(v, idx);
_mm256_storeu_si256(reinterpret_cast<__m256i*>(o + w), packed); w += static_cast<std::size_t>(popcnt_u32(bits));
}
}
for (; i < n; ++i)
if (p[i] > thr) o[w++] = p[i];
return w;
}
}