base64-ng 2.0.0

#![allow(unsafe_code)]

//! SIMD admission boundary.
//!
//! This module is the only source module allowed to lower the crate-level
//! `unsafe_code` lint. Keep all future architecture-specific intrinsics behind
//! this boundary, with a local safety explanation for every unsafe block.
//!
//! The module admits `x86`/`x86_64` AVX-512 VBMI, AVX2, SSSE3/SSE4.1,
//! and little-endian `aarch64` NEON encode backends for Standard and
//! URL-safe alphabet families. It also admits `x86`/`x86_64` AVX-512 VBMI,
//! AVX2, and SSSE3/SSE4.1 strict decode plus little-endian `aarch64` NEON
//! strict decode for Standard and URL-safe alphabet families through the
//! separate decode backend boundary. Narrow wasm `simd128` encode and strict
//! decode are admitted for the documented wasm runtime profile. Wrapped decode
//! may enter admitted strict decode only after scalar line-profile validation
//! and line-ending compaction. Legacy whitespace decode may enter admitted
//! strict decode only after scalar whitespace compaction. Strict in-place
//! encode and decode may enter admitted backends only after stack staging.
//! Exact-profile Linux RISC-V dispatch may use RVV 1.0 on the measured
//! `SpacemiT` X60 profile. Non-Standard-family custom alphabets, other RISC-V
//! profiles, big-endian `AArch64`, CT secret decode, and every other unsupported
//! SIMD surface still execute through the scalar implementation.
//! `Alphabet::ENCODE` is authoritative for ordinary scalar and
//! SIMD decode; overridable `Alphabet::decode` code is never an admission input.
//! A `no_std` build may execute an admitted backend only when complete static
//! target-feature evidence and the atomic backend-health latch are available.
//!
//! The x86 AVX-512 VBMI, AVX2, SSSE3/SSE4.1, and `AArch64` NEON fixed-block
//! encoders are reachable after runtime CPU probing on `std`, or complete
//! compile-time target-feature checks on `no_std`. The wasm `simd128`
//! fixed-block implementation is reachable only for the admitted wasm runtime
//! profile and remains scoped out of broader browser/JIT claims.
//! Commit 33 also carries an internal, non-dispatchable SVE encode/decode
//! candidate for QEMU and generated-code evidence. Public `AArch64` dispatch
//! remains admitted NEON or scalar until the native SVE admission contract is
//! satisfied.

mod static_token;
pub use static_token::StaticBackendToken;

#[cfg(all(
    feature = "simd",
    target_arch = "aarch64",
    target_endian = "little",
    base64_ng_sve_candidate
))]
mod sve;
#[cfg(all(
    feature = "std",
    feature = "simd",
    test,
    target_arch = "aarch64",
    target_endian = "little",
    target_os = "linux",
    base64_ng_sve_candidate
))]
mod sve_tests;

#[cfg(all(feature = "simd", target_arch = "riscv64"))]
mod rvv;
#[cfg(all(feature = "simd", target_arch = "riscv64", base64_ng_perf_evidence))]
pub(crate) use rvv::candidate_available as rvv_candidate_available;
#[cfg(all(feature = "simd", target_arch = "riscv64"))]
pub(crate) use rvv::{
    available as rvv_available, decode_slice as decode_slice_rvv, encode_slice as encode_slice_rvv,
    supports_alphabet as rvv_supports_alphabet,
};
#[cfg(all(
    feature = "std",
    feature = "simd",
    test,
    target_arch = "riscv64",
    base64_ng_rvv_candidate
))]
mod rvv_tests;

#[cfg(any(target_arch = "aarch64", target_arch = "arm"))]
mod neon;
#[cfg(all(test, target_arch = "aarch64", target_endian = "little"))]
pub(in crate::simd) use neon::decode_16_bytes_neon;
#[cfg(all(feature = "simd", target_arch = "aarch64", target_endian = "little"))]
pub(crate) use neon::decode_slice_neon;
#[cfg(all(
    test,
    any(
        target_arch = "aarch64",
        all(target_arch = "arm", target_feature = "neon")
    )
))]
pub(in crate::simd) use neon::encode_12_bytes_neon;
#[cfg(all(feature = "simd", target_arch = "aarch64", target_endian = "little"))]
pub(crate) use neon::encode_slice_neon;
#[cfg(any(target_arch = "aarch64", target_arch = "arm"))]
pub(crate) use neon::neon_available;
#[cfg(all(feature = "simd", target_arch = "aarch64", target_endian = "little"))]
pub(crate) use neon::neon_supports_alphabet;
#[cfg(all(feature = "simd", target_arch = "aarch64", target_endian = "little"))]
pub(crate) use neon::neon_supports_decode_alphabet;
#[cfg(any(
    all(test, any(target_arch = "x86", target_arch = "x86_64")),
    all(feature = "simd", any(target_arch = "x86", target_arch = "x86_64"))
))]
mod x86;
#[cfg(all(
    feature = "std",
    test,
    any(target_arch = "x86", target_arch = "x86_64")
))]
pub(super) use x86::{
    decode_16_bytes_ssse3_sse41, decode_32_bytes_avx2, decode_64_bytes_avx512,
    encode_12_bytes_ssse3_sse41, encode_24_bytes_avx2, encode_48_bytes_avx512,
};
#[cfg(all(
    feature = "std",
    test,
    target_arch = "aarch64",
    target_endian = "little"
))]
mod neon_decode_tests;
#[cfg(all(
    feature = "std",
    test,
    target_arch = "aarch64",
    target_endian = "little"
))]
mod neon_direct_tests;
#[cfg(all(target_arch = "wasm32", any(test, feature = "simd")))]
mod wasm;
#[cfg(all(
    feature = "std",
    test,
    any(target_arch = "x86", target_arch = "x86_64")
))]
mod x86_decode_direct_tests;
#[cfg(all(
    feature = "std",
    test,
    any(target_arch = "x86", target_arch = "x86_64")
))]
mod x86_decode_tests;
#[cfg(all(
    feature = "std",
    test,
    any(target_arch = "x86", target_arch = "x86_64")
))]
mod x86_encode_tests;

/// Backend currently allowed to execute.
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub(crate) enum ActiveBackend {
    /// The audited scalar implementation.
    Scalar,
    /// `x86`/`x86_64` AVX-512 VBMI encode backend.
    #[cfg(all(feature = "simd", any(target_arch = "x86", target_arch = "x86_64")))]
    Avx512Vbmi,
    /// `x86`/`x86_64` AVX2 encode backend.
    #[cfg(all(feature = "simd", any(target_arch = "x86", target_arch = "x86_64")))]
    Avx2,
    /// `x86`/`x86_64` SSSE3/SSE4.1 encode backend.
    #[cfg(all(feature = "simd", any(target_arch = "x86", target_arch = "x86_64")))]
    Ssse3Sse41,
    /// Little-endian `aarch64` NEON encode backend.
    #[cfg(all(feature = "simd", target_arch = "aarch64", target_endian = "little"))]
    Neon,
    /// wasm32 `simd128` fixed-block encode backend.
    #[cfg(all(feature = "simd", target_arch = "wasm32"))]
    WasmSimd128,
    /// Linux `SpacemiT` X60 RVV 1.0 encode backend.
    #[cfg(all(
        feature = "std",
        feature = "simd",
        target_arch = "riscv64",
        target_os = "linux"
    ))]
    Rvv,
}

/// SIMD candidate detected for the current target.
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub(crate) enum Candidate {
    /// No supported SIMD candidate was detected.
    Scalar,
    /// `x86`/`x86_64` AVX-512 VBMI is available.
    #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
    Avx512Vbmi,
    /// `x86`/`x86_64` AVX2 is available.
    #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
    Avx2,
    /// `x86`/`x86_64` SSSE3/SSE4.1 is available.
    #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
    Ssse3Sse41,
    /// ARM NEON is available.
    #[cfg(any(target_arch = "aarch64", target_arch = "arm"))]
    Neon,
    /// SVE is visible to the non-admitted project-owned candidate build.
    #[cfg(all(target_arch = "aarch64", base64_ng_sve_candidate))]
    Sve,
    /// wasm32 `simd128` is available.
    #[cfg(target_arch = "wasm32")]
    WasmSimd128,
    /// RVV is visible to the exact production profile or candidate build.
    #[cfg(target_arch = "riscv64")]
    Rvv,
}

/// Returns the backend that is allowed to execute for this build.
#[must_use]
pub(crate) fn active_backend() -> ActiveBackend {
    #[cfg(all(feature = "std", not(target_arch = "riscv64")))]
    {
        static ACTIVE_BACKEND: std::sync::OnceLock<ActiveBackend> = std::sync::OnceLock::new();
        *ACTIVE_BACKEND.get_or_init(detect_active_backend)
    }

    #[cfg(any(not(feature = "std"), target_arch = "riscv64"))]
    {
        detect_active_backend()
    }
}

fn detect_active_backend() -> ActiveBackend {
    #[cfg(all(feature = "simd", any(target_arch = "x86", target_arch = "x86_64")))]
    {
        if avx512_vbmi_base64_available() {
            return ActiveBackend::Avx512Vbmi;
        }

        if avx2_available() {
            return ActiveBackend::Avx2;
        }

        if ssse3_sse41_available() {
            return ActiveBackend::Ssse3Sse41;
        }
    }

    #[cfg(all(feature = "simd", target_arch = "aarch64", target_endian = "little"))]
    {
        if neon_available() {
            return ActiveBackend::Neon;
        }
    }

    #[cfg(all(feature = "simd", target_arch = "wasm32"))]
    {
        if wasm_simd128_available() {
            return ActiveBackend::WasmSimd128;
        }
    }

    #[cfg(all(
        feature = "std",
        feature = "simd",
        target_arch = "riscv64",
        target_os = "linux"
    ))]
    {
        if rvv::available() {
            return ActiveBackend::Rvv;
        }
    }

    let _ = detected_candidate();
    ActiveBackend::Scalar
}

/// Returns the fastest SIMD candidate visible to this build.
///
/// Candidate detection is intentionally separate from activation. Only the
/// admitted target and operation scopes may become active; every other SIMD
/// candidate still executes through scalar code.
#[must_use]
pub(crate) fn detected_candidate() -> Candidate {
    #[cfg(all(
        feature = "simd",
        target_arch = "aarch64",
        target_endian = "little",
        base64_ng_sve_candidate
    ))]
    {
        if sve::available() {
            return Candidate::Sve;
        }
    }

    #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
    {
        if avx512_vbmi_base64_available() {
            return Candidate::Avx512Vbmi;
        }

        if avx2_available() {
            return Candidate::Avx2;
        }

        if ssse3_sse41_available() {
            return Candidate::Ssse3Sse41;
        }
    }

    #[cfg(any(target_arch = "aarch64", target_arch = "arm"))]
    {
        if neon_available() {
            return Candidate::Neon;
        }
    }

    #[cfg(target_arch = "wasm32")]
    {
        if wasm_simd128_available() {
            return Candidate::WasmSimd128;
        }
    }

    #[cfg(all(feature = "simd", target_arch = "riscv64", base64_ng_rvv_candidate))]
    {
        if rvv::candidate_available() {
            return Candidate::Rvv;
        }
    }

    #[cfg(all(
        feature = "std",
        feature = "simd",
        target_arch = "riscv64",
        target_os = "linux",
        not(base64_ng_rvv_candidate)
    ))]
    {
        if rvv::available() {
            return Candidate::Rvv;
        }
    }

    Candidate::Scalar
}

#[cfg(all(feature = "simd", any(target_arch = "x86", target_arch = "x86_64")))]
pub(crate) use x86::{
    avx2_decode_available, avx2_encode_available, avx2_supports_alphabet,
    avx2_supports_decode_alphabet, avx512_decode_available, avx512_supports_alphabet,
    avx512_supports_decode_alphabet, encode_slice_avx2, encode_slice_avx512,
    encode_slice_ssse3_sse41, ssse3_sse41_decode_available, ssse3_sse41_encode_available,
    ssse3_sse41_supports_alphabet, ssse3_sse41_supports_decode_alphabet,
};

#[cfg(all(feature = "simd", any(target_arch = "x86", target_arch = "x86_64")))]
pub(crate) use x86::{decode_slice_avx2, decode_slice_avx512, decode_slice_ssse3_sse41};

#[cfg(all(feature = "simd", target_arch = "wasm32"))]
pub(crate) use wasm::{decode_slice_wasm_simd128, encode_slice_wasm_simd128};

#[cfg(all(feature = "simd", target_arch = "wasm32"))]
pub(crate) use wasm::{
    wasm_simd128_decode_available, wasm_simd128_supports_alphabet,
    wasm_simd128_supports_decode_alphabet,
};

#[cfg(all(feature = "std", any(target_arch = "x86", target_arch = "x86_64")))]
fn avx512_vbmi_base64_available() -> bool {
    std::is_x86_feature_detected!("avx512f")
        && std::is_x86_feature_detected!("avx512bw")
        && std::is_x86_feature_detected!("avx512vl")
        && std::is_x86_feature_detected!("avx512vbmi")
}

#[cfg(all(not(feature = "std"), any(target_arch = "x86", target_arch = "x86_64")))]
fn avx512_vbmi_base64_available() -> bool {
    cfg!(target_feature = "avx512f")
        && cfg!(target_feature = "avx512bw")
        && cfg!(target_feature = "avx512vl")
        && cfg!(target_feature = "avx512vbmi")
}

#[cfg(all(feature = "std", any(target_arch = "x86", target_arch = "x86_64")))]
fn avx2_available() -> bool {
    std::is_x86_feature_detected!("avx2")
}

#[cfg(all(not(feature = "std"), any(target_arch = "x86", target_arch = "x86_64")))]
fn avx2_available() -> bool {
    cfg!(target_feature = "avx2")
}

#[cfg(all(feature = "std", any(target_arch = "x86", target_arch = "x86_64")))]
fn ssse3_sse41_available() -> bool {
    std::is_x86_feature_detected!("ssse3") && std::is_x86_feature_detected!("sse4.1")
}

#[cfg(all(not(feature = "std"), any(target_arch = "x86", target_arch = "x86_64")))]
fn ssse3_sse41_available() -> bool {
    cfg!(target_feature = "ssse3") && cfg!(target_feature = "sse4.1")
}

#[cfg(target_arch = "wasm32")]
fn wasm_simd128_available() -> bool {
    cfg!(target_feature = "simd128")
}

#[cfg(target_arch = "wasm32")]
pub(crate) const fn wasm_simd128_artifact_selected() -> bool {
    cfg!(target_feature = "simd128")
}

#[cfg(all(
    test,
    any(
        target_arch = "aarch64",
        all(feature = "std", any(target_arch = "x86", target_arch = "x86_64"))
    )
))]
mod tests;