use core::sync::atomic::{AtomicU8, Ordering};
pub mod scalar;
#[cfg(target_arch = "aarch64")]
pub mod neon;
#[cfg(target_arch = "x86_64")]
pub mod avx2;
#[cfg(target_arch = "x86_64")]
pub mod avx2x12;
#[cfg(target_arch = "x86_64")]
pub mod avx512;
#[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
pub mod sse41;
#[cfg(all(target_arch = "wasm32", target_feature = "simd128"))]
pub mod wasm128;
#[derive(Copy, Clone, PartialEq, Eq, Debug, Hash, PartialOrd, Ord)]
#[repr(u8)]
#[non_exhaustive]
pub enum Backend {
Scalar = 0,
Neon = 1,
Sse41 = 2,
Avx2 = 3,
Avx512 = 4,
Wasm128 = 5,
Avx2x12 = 6,
}
pub type BcryptLanesFn = unsafe fn(u32, &[[u32; 18]], &[[u32; 4]], &mut [[u8; 24]]);
impl Backend {
pub const ALL: &'static [Backend] = &[
Backend::Scalar,
Backend::Neon,
Backend::Sse41,
Backend::Avx2,
Backend::Avx2x12,
Backend::Avx512,
Backend::Wasm128,
];
#[inline]
#[must_use]
pub const fn name(self) -> &'static str {
match self {
Backend::Scalar => "scalar",
Backend::Neon => "neon",
Backend::Sse41 => "sse41",
Backend::Avx2 => "avx2",
Backend::Avx512 => "avx512",
Backend::Wasm128 => "wasm128",
Backend::Avx2x12 => "avx2_12",
}
}
#[inline]
#[must_use]
pub const fn lanes(self) -> usize {
match self {
Backend::Scalar => 1,
Backend::Neon => 8,
Backend::Sse41 => 4,
Backend::Avx2 => 8,
Backend::Avx512 => 16,
Backend::Avx2x12 => 12,
#[cfg(all(target_arch = "wasm32", target_feature = "simd128"))]
Backend::Wasm128 => wasm128::LANES,
#[cfg(not(all(target_arch = "wasm32", target_feature = "simd128")))]
Backend::Wasm128 => 8,
}
}
#[inline]
#[must_use]
pub fn is_available(self) -> bool {
match self {
Backend::Scalar => true,
Backend::Neon => have_neon(),
Backend::Sse41 => have_sse41(),
Backend::Avx2 => have_avx2(),
Backend::Avx512 => have_avx512f(),
Backend::Avx2x12 => have_avx2(),
Backend::Wasm128 => have_wasm_simd128(),
}
}
#[inline]
const fn to_u8(self) -> u8 {
self as u8
}
#[inline]
const fn from_u8(value: u8) -> Backend {
match value {
1 => Backend::Neon,
2 => Backend::Sse41,
3 => Backend::Avx2,
4 => Backend::Avx512,
5 => Backend::Wasm128,
6 => Backend::Avx2x12,
_ => Backend::Scalar,
}
}
}
impl core::fmt::Display for Backend {
fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
f.write_str(self.name())
}
}
#[cfg(all(feature = "std", target_arch = "x86_64"))]
#[inline]
fn have_avx512f() -> bool {
std::arch::is_x86_feature_detected!("avx512f")
}
#[cfg(not(all(feature = "std", target_arch = "x86_64")))]
#[inline]
fn have_avx512f() -> bool {
cfg!(all(target_arch = "x86_64", target_feature = "avx512f"))
}
#[cfg(all(feature = "std", target_arch = "x86_64"))]
#[inline]
fn have_avx2() -> bool {
std::arch::is_x86_feature_detected!("avx2")
}
#[cfg(not(all(feature = "std", target_arch = "x86_64")))]
#[inline]
fn have_avx2() -> bool {
cfg!(all(target_arch = "x86_64", target_feature = "avx2"))
}
#[cfg(all(feature = "std", any(target_arch = "x86", target_arch = "x86_64")))]
#[inline]
fn have_sse41() -> bool {
std::arch::is_x86_feature_detected!("sse4.1")
}
#[cfg(not(all(feature = "std", any(target_arch = "x86", target_arch = "x86_64"))))]
#[inline]
fn have_sse41() -> bool {
cfg!(all(
any(target_arch = "x86", target_arch = "x86_64"),
target_feature = "sse4.1"
))
}
#[cfg(all(feature = "std", target_arch = "aarch64"))]
#[inline]
fn have_neon() -> bool {
#[cfg(any(target_vendor = "apple", target_os = "windows"))]
{
true
}
#[cfg(not(any(target_vendor = "apple", target_os = "windows")))]
{
std::arch::is_aarch64_feature_detected!("neon")
}
}
#[cfg(not(all(feature = "std", target_arch = "aarch64")))]
#[inline]
fn have_neon() -> bool {
cfg!(all(target_arch = "aarch64", target_feature = "neon"))
}
#[cfg(all(target_arch = "wasm32", target_feature = "simd128"))]
#[inline]
fn have_wasm_simd128() -> bool {
true
}
#[cfg(not(all(target_arch = "wasm32", target_feature = "simd128")))]
#[inline]
fn have_wasm_simd128() -> bool {
false
}
const UNINIT: u8 = 0xFF;
static CACHED_BACKEND: AtomicU8 = AtomicU8::new(UNINIT);
#[must_use]
pub fn detect() -> Backend {
if cfg!(miri) {
return Backend::Scalar;
}
#[cfg(feature = "std")]
if let Some(forced) = forced_backend() {
return forced;
}
if have_avx512f() || have_avx2() {
width_pick()
} else if have_sse41() {
Backend::Sse41
} else if have_neon() {
Backend::Neon
} else if have_wasm_simd128() {
Backend::Wasm128
} else {
Backend::Scalar
}
}
#[cfg(feature = "std")]
fn backend_by_name(name: &str) -> Option<Backend> {
Backend::ALL.iter().copied().find(|b| b.name() == name)
}
#[cfg(feature = "std")]
fn forced_backend() -> Option<Backend> {
let raw = std::env::var_os("BCRYPT_FORCE_BACKEND")?;
let name = raw
.to_str()
.unwrap_or_else(|| panic!("BCRYPT_FORCE_BACKEND is not UTF-8"));
let backend = backend_by_name(name).unwrap_or_else(|| {
let valid = Backend::ALL
.iter()
.map(|b| b.name())
.collect::<std::vec::Vec<_>>()
.join(", ");
panic!("BCRYPT_FORCE_BACKEND={name}: unknown backend (valid: {valid})");
});
assert!(
backend.is_available(),
"BCRYPT_FORCE_BACKEND={name} but {name} is not available on this host"
);
Some(backend)
}
#[cold]
#[inline(never)]
fn detect_and_cache() -> Backend {
let detected = detect();
CACHED_BACKEND.store(detected.to_u8(), Ordering::Relaxed);
detected
}
#[inline]
#[must_use]
pub fn backend() -> Backend {
let cached = CACHED_BACKEND.load(Ordering::Relaxed);
if cached == UNINIT {
detect_and_cache()
} else {
Backend::from_u8(cached)
}
}
#[cfg(all(feature = "std", target_arch = "x86_64", not(debug_assertions), not(miri)))]
static WIDTH_PICK: std::sync::OnceLock<u8> = std::sync::OnceLock::new();
#[cfg(all(feature = "std", target_arch = "x86_64", not(debug_assertions), not(miri)))]
#[allow(unused_unsafe)]
fn l1d_size() -> Option<usize> {
use core::arch::x86_64::{__cpuid, __cpuid_count};
if unsafe { __cpuid(0) }.eax < 4 {
return None;
}
for sub_leaf in 0..16u32 {
let r = unsafe { __cpuid_count(4, sub_leaf) };
let cache_type = r.eax & 0x1f;
if cache_type == 0 {
break;
}
if cache_type == 1 && (r.eax >> 5) & 0x7 == 1 {
let line_size = (r.ebx & 0xfff) as usize + 1;
let partitions = ((r.ebx >> 12) & 0x3ff) as usize + 1;
let ways = ((r.ebx >> 22) & 0x3ff) as usize + 1;
let sets = r.ecx as usize + 1;
return Some(line_size * partitions * ways * sets);
}
}
None
}
#[cfg(all(feature = "std", target_arch = "x86_64", not(debug_assertions), not(miri)))]
fn width_pick() -> Backend {
let avx512 = have_avx512f();
let avx2 = have_avx2();
let x12 = avx2 && l1d_size().is_some_and(|bytes| bytes >= 48 * 1024);
if !avx2 {
return Backend::Avx512;
}
if !avx512 && !x12 {
return Backend::Avx2;
}
Backend::from_u8(*WIDTH_PICK.get_or_init(|| width_shootout(avx512, x12).to_u8()))
}
#[cfg(not(all(feature = "std", target_arch = "x86_64", not(debug_assertions), not(miri))))]
fn width_pick() -> Backend {
if have_avx512f() {
Backend::Avx512
} else {
Backend::Avx2
}
}
#[cfg(all(feature = "std", target_arch = "x86_64", not(debug_assertions), not(miri)))]
fn width_shootout(has_avx512: bool, has_x12: bool) -> Backend {
use std::time::Instant;
let mut rng = 0xBC79_7A5A_F1A0_0003u64;
let mut next_u32 = move || {
rng = rng.wrapping_add(0x9E37_79B9_7F4A_7C15);
let mut z = rng;
z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9);
z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB);
(z ^ (z >> 31)) as u32
};
const ITEMS: usize = 48;
let mut kws = [[0u32; 18]; ITEMS];
let mut sws = [[0u32; 4]; ITEMS];
for w in kws.as_flattened_mut().iter_mut().chain(sws.as_flattened_mut()) {
*w = next_u32();
}
fn run_batch(
kernel: BcryptLanesFn,
lanes: usize,
kws: &[[u32; 18]; ITEMS],
sws: &[[u32; 4]; ITEMS],
outs: &mut [[u8; 24]; ITEMS],
) {
let mut base = 0;
while base < ITEMS {
unsafe {
kernel(
4,
&kws[base..base + lanes],
&sws[base..base + lanes],
&mut outs[base..base + lanes],
)
};
base += lanes;
}
}
let mut arms = [(Backend::Avx2, bcrypt_lanes_fn(Backend::Avx2)); 3];
let mut n = 0;
if has_avx512 {
arms[n] = (Backend::Avx512, bcrypt_lanes_fn(Backend::Avx512));
n += 1;
}
n += 1;
if has_x12 {
arms[n] = (Backend::Avx2x12, bcrypt_lanes_fn(Backend::Avx2x12));
n += 1;
}
let arms = &arms[..n];
let mut outs = [[[0u8; 24]; ITEMS]; 3];
for (i, &(backend, kernel)) in arms.iter().enumerate() {
run_batch(kernel, backend.lanes(), &kws, &sws, &mut outs[i]);
assert_eq!(
outs[i], outs[0],
"width shootout: {} and {} kernels diverged — a kernel bug, not timing",
backend.name(),
arms[0].0.name(),
);
}
let mut best = [f64::MAX; 3];
for _ in 0..3 {
for (i, &(backend, kernel)) in arms.iter().enumerate() {
let start = Instant::now();
run_batch(kernel, backend.lanes(), &kws, &sws, &mut outs[i]);
best[i] = best[i].min(start.elapsed().as_secs_f64());
core::hint::black_box(&mut outs[i]);
}
}
let (mut pick, mut pick_best) = (arms[0].0, best[0]);
for (i, &(backend, _)) in arms.iter().enumerate().skip(1) {
if best[i] < pick_best {
pick = backend;
pick_best = best[i];
}
}
pick
}
#[must_use]
pub fn bcrypt_lanes_fn(backend: Backend) -> BcryptLanesFn {
match backend {
Backend::Scalar => scalar::bcrypt_lanes,
#[cfg(target_arch = "aarch64")]
Backend::Neon => neon::bcrypt_lanes,
#[cfg(not(target_arch = "aarch64"))]
Backend::Neon => scalar::bcrypt_lanes,
#[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
Backend::Sse41 => sse41::bcrypt_lanes,
#[cfg(not(any(target_arch = "x86", target_arch = "x86_64")))]
Backend::Sse41 => scalar::bcrypt_lanes,
#[cfg(target_arch = "x86_64")]
Backend::Avx2 => avx2::bcrypt_lanes,
#[cfg(not(target_arch = "x86_64"))]
Backend::Avx2 => scalar::bcrypt_lanes,
#[cfg(target_arch = "x86_64")]
Backend::Avx512 => avx512::bcrypt_lanes,
#[cfg(not(target_arch = "x86_64"))]
Backend::Avx512 => scalar::bcrypt_lanes,
#[cfg(target_arch = "x86_64")]
Backend::Avx2x12 => avx2x12::bcrypt_lanes,
#[cfg(not(target_arch = "x86_64"))]
Backend::Avx2x12 => scalar::bcrypt_lanes,
#[cfg(all(target_arch = "wasm32", target_feature = "simd128"))]
Backend::Wasm128 => wasm128::bcrypt_lanes,
#[cfg(not(all(target_arch = "wasm32", target_feature = "simd128")))]
Backend::Wasm128 => scalar::bcrypt_lanes,
}
}
#[must_use]
pub(crate) fn expand_key_words(key: &[u8; 72], key_len: usize) -> [u32; 18] {
debug_assert!((1..=72).contains(&key_len));
let mut words = [0u32; 18];
let mut j = 0usize;
for w in &mut words {
let mut word = 0u32;
for _ in 0..4 {
if j >= key_len {
j = 0;
}
word = (word << 8) | u32::from(key[j]);
j += 1;
}
*w = word;
}
words
}
#[must_use]
pub(crate) fn salt_words(salt: &[u8; 16]) -> [u32; 4] {
let mut words = [0u32; 4];
for (w, chunk) in words.iter_mut().zip(salt.chunks_exact(4)) {
*w = u32::from_be_bytes([chunk[0], chunk[1], chunk[2], chunk[3]]);
}
words
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn backend_u8_round_trip() {
for &b in Backend::ALL {
assert_eq!(Backend::from_u8(b.to_u8()), b);
}
assert_eq!(Backend::from_u8(UNINIT), Backend::Scalar);
assert_eq!(Backend::from_u8(200), Backend::Scalar);
}
#[test]
fn cache_agrees_with_detect() {
let first = backend();
assert_eq!(first, detect());
assert_eq!(backend(), first);
assert_ne!(CACHED_BACKEND.load(Ordering::Relaxed), UNINIT);
}
#[cfg(feature = "std")]
#[test]
fn backend_by_name_round_trips_every_variant() {
for &b in Backend::ALL {
assert_eq!(backend_by_name(b.name()), Some(b));
}
assert_eq!(backend_by_name("AVX2"), None);
assert_eq!(backend_by_name("avx-512"), None);
assert_eq!(backend_by_name(""), None);
}
#[test]
fn width_pick_is_stable_and_cached() {
#[cfg(feature = "std")]
if std::env::var_os("BCRYPT_FORCE_BACKEND").is_some() {
return;
}
if cfg!(miri) || !Backend::Avx2.is_available() {
return;
}
let first = detect();
assert!(matches!(
first,
Backend::Avx2 | Backend::Avx512 | Backend::Avx2x12
));
assert_eq!(detect(), first);
#[cfg(all(feature = "std", target_arch = "x86_64", not(debug_assertions), not(miri)))]
if Backend::Avx512.is_available() || l1d_size().is_some_and(|bytes| bytes >= 48 * 1024) {
assert!(WIDTH_PICK.get().is_some());
}
}
#[test]
fn detected_backend_is_available() {
assert!(detect().is_available());
assert!(Backend::Scalar.is_available());
}
#[test]
fn detection_respects_the_architecture() {
#[cfg(feature = "std")]
if std::env::var_os("BCRYPT_FORCE_BACKEND").is_some() {
return;
}
if cfg!(miri) {
assert_eq!(detect(), Backend::Scalar);
return;
}
if cfg!(target_arch = "aarch64") {
assert!(!have_sse41());
assert!(!have_avx2());
assert!(!have_avx512f());
if cfg!(target_vendor = "apple") {
assert_eq!(detect(), Backend::Neon);
} else {
assert!(matches!(detect(), Backend::Neon | Backend::Scalar));
}
}
if cfg!(target_arch = "x86_64") {
assert!(!have_neon());
assert!(matches!(
detect(),
Backend::Sse41
| Backend::Avx2
| Backend::Avx512
| Backend::Avx2x12
| Backend::Scalar
));
}
if cfg!(target_arch = "wasm32") {
assert!(!have_sse41());
assert!(!have_avx2());
assert!(!have_avx512f());
assert!(!have_neon());
if cfg!(target_feature = "simd128") {
assert_eq!(detect(), Backend::Wasm128);
} else {
assert_eq!(detect(), Backend::Scalar);
}
}
}
#[test]
fn every_backend_resolves_to_a_function() {
for &b in Backend::ALL {
let f = bcrypt_lanes_fn(b);
if b == Backend::Scalar && !cfg!(miri) {
let scalar = bcrypt_lanes_fn(Backend::Scalar);
assert!(core::ptr::fn_addr_eq(f, scalar));
}
}
}
#[test]
fn expand_key_words_cycles_big_endian() {
let mut key = [0u8; 72];
key[0] = b'a';
let w = expand_key_words(&key, 2);
assert_eq!(w[0], 0x61006100);
assert_eq!(w[17], 0x61006100);
let key72: [u8; 72] = core::array::from_fn(|i| i as u8);
let w = expand_key_words(&key72, 72);
assert_eq!(w[0], 0x00010203);
assert_eq!(w[17], 0x44454647);
let w = expand_key_words(&key72, 71);
assert_eq!(w[17], 0x44454600);
}
#[test]
fn salt_words_are_big_endian() {
let salt: [u8; 16] = core::array::from_fn(|i| (i + 1) as u8);
assert_eq!(
salt_words(&salt),
[0x01020304, 0x05060708, 0x090A0B0C, 0x0D0E0F10]
);
}
}