#include "sve/sve.hpp"
#if defined(SVE_X86)
# if defined(_MSC_VER)
# include <intrin.h>
# include <immintrin.h>
# else
# include <cpuid.h>
# include <immintrin.h>
# endif
#endif
#if defined(SVE_ARM) && defined(__linux__)
# include <sys/auxv.h>
#endif
namespace sve {
namespace {
#if defined(SVE_X86)
struct Regs { std::uint32_t eax, ebx, ecx, edx; };
inline Regs cpuid(std::uint32_t leaf, std::uint32_t subleaf) noexcept {
Regs r{};
#if defined(_MSC_VER)
int out[4];
__cpuidex(out, static_cast<int>(leaf), static_cast<int>(subleaf));
r = {static_cast<std::uint32_t>(out[0]), static_cast<std::uint32_t>(out[1]),
static_cast<std::uint32_t>(out[2]), static_cast<std::uint32_t>(out[3])};
#else
__cpuid_count(leaf, subleaf, r.eax, r.ebx, r.ecx, r.edx);
#endif
return r;
}
inline std::uint64_t xgetbv0() noexcept {
#if defined(_MSC_VER)
return _xgetbv(0);
#else
std::uint32_t lo, hi;
__asm__ __volatile__("xgetbv" : "=a"(lo), "=d"(hi) : "c"(0));
return (static_cast<std::uint64_t>(hi) << 32) | lo;
#endif
}
#endif
CpuFeatures detect() noexcept {
CpuFeatures f{};
#if defined(SVE_X86)
const Regs max = cpuid(0, 0);
if (max.eax < 1) return f;
const Regs l1 = cpuid(1, 0);
f.sse2 = (l1.edx & (1u << 26)) != 0;
f.sse42 = (l1.ecx & (1u << 20)) != 0;
f.popcnt = (l1.ecx & (1u << 23)) != 0;
const bool osxsave = (l1.ecx & (1u << 27)) != 0; const bool cpu_avx = (l1.ecx & (1u << 28)) != 0;
bool os_ymm = false, os_zmm = false;
if (osxsave) {
const std::uint64_t xcr0 = xgetbv0();
os_ymm = (xcr0 & 0x6) == 0x6;
os_zmm = os_ymm && (xcr0 & 0xE0) == 0xE0;
}
f.avx = cpu_avx && os_ymm;
if (max.eax >= 7) {
const Regs l7 = cpuid(7, 0);
f.avx2 = os_ymm && (l7.ebx & (1u << 5)) != 0;
f.avx512f = os_zmm && (l7.ebx & (1u << 16)) != 0;
f.avx512dq = os_zmm && (l7.ebx & (1u << 17)) != 0;
f.avx512bw = os_zmm && (l7.ebx & (1u << 30)) != 0;
f.avx512vl = os_zmm && (l7.ebx & (1u << 31)) != 0;
}
f.fma = f.avx && (l1.ecx & (1u << 12)) != 0;
#endif
#if defined(SVE_ARM)
f.neon = true;
#endif
return f;
}
}
std::vector<std::string> CpuFeatures::enabled() const {
std::vector<std::string> v;
auto push = [&](bool on, const char* n) { if (on) v.emplace_back(n); };
push(sse2, "sse2"); push(sse42, "sse4.2"); push(popcnt, "popcnt");
push(avx, "avx"); push(avx2, "avx2"); push(fma, "fma");
push(avx512f, "avx512f"); push(avx512bw, "avx512bw");
push(avx512dq, "avx512dq"); push(avx512vl, "avx512vl");
push(neon, "neon");
return v;
}
const CpuFeatures& features() noexcept {
static const CpuFeatures f = detect();
return f;
}
std::vector<Isa> available_isas() {
const auto& f = features();
std::vector<Isa> v{Isa::Scalar};
if (f.neon) v.push_back(Isa::Neon);
if (f.avx2) v.push_back(Isa::Avx2);
if (f.avx512f) v.push_back(Isa::Avx512);
return v;
}
Isa best_isa() noexcept {
static const Isa cached = [] {
const auto isas = available_isas();
if (const char* e = std::getenv("SVE_ISA")) {
std::string s{e};
Isa want = Isa::Scalar;
bool ok = true;
if (s == "scalar") want = Isa::Scalar;
else if (s == "neon") want = Isa::Neon;
else if (s == "avx2") want = Isa::Avx2;
else if (s == "avx512") want = Isa::Avx512;
else ok = false;
if (ok) for (Isa i : isas) if (i == want) return want;
}
return isas.back();
}();
return cached;
}
}