use core::arch::x86_64::{
__m128i, __m256i, _mm_add_epi32, _mm_and_si128, _mm_extract_epi32, _mm_insert_epi32,
_mm_loadu_si128, _mm_set1_epi32, _mm_setr_epi32, _mm_setzero_si128, _mm_slli_epi32,
_mm_srli_epi32, _mm_store_si128, _mm_storeu_si128, _mm_xor_si128, _mm256_add_epi32,
_mm256_and_si256, _mm256_extract_epi32, _mm256_insert_epi32, _mm256_loadu_si256,
_mm256_set1_epi32, _mm256_setr_epi32, _mm256_setzero_si256, _mm256_slli_epi32,
_mm256_srli_epi32, _mm256_store_si256, _mm256_storeu_si256, _mm256_xor_si256,
};
use core::sync::atomic::{AtomicU8, Ordering};
use crate::consts::{P_INIT, S_INIT};
const LANES: usize = 12;
#[repr(C, align(64))]
struct SBoxes8([[u32; 8]; 1024]);
impl SBoxes8 {
#[inline(always)]
fn box_base(&self, b: usize) -> *const u32 {
debug_assert!(b < 4);
unsafe { self.0.as_ptr().cast::<u32>().add(b * 256 * 8) }
}
}
#[repr(C, align(64))]
struct SBoxes4([[u32; 4]; 1024]);
impl SBoxes4 {
#[inline(always)]
fn box_base(&self, b: usize) -> *const u32 {
debug_assert!(b < 4);
unsafe { self.0.as_ptr().cast::<u32>().add(b * 256 * 4) }
}
}
struct State12 {
p8: [__m256i; 18],
p4: [__m128i; 18],
s8: SBoxes8,
s4: SBoxes4,
}
macro_rules! lane_load {
($base:expr, $byte_off:expr) => {{
let mut a = $base as usize + $byte_off;
core::arch::asm!(
"/* {0} */",
inout(reg) a,
options(pure, nomem, nostack, preserves_flags),
);
*(a as *const u32)
}};
}
macro_rules! lookup8_insert {
($box_base:expr, $off:expr) => {{
let base = ($box_base).cast::<u8>();
let mut offs = [0u32; 8];
unsafe { _mm256_storeu_si256(offs.as_mut_ptr().cast::<__m256i>(), $off) };
let gathered = [
unsafe { lane_load!(base, offs[0] as usize) },
unsafe { lane_load!(base, offs[1] as usize + 4) },
unsafe { lane_load!(base, offs[2] as usize + 8) },
unsafe { lane_load!(base, offs[3] as usize + 12) },
unsafe { lane_load!(base, offs[4] as usize + 16) },
unsafe { lane_load!(base, offs[5] as usize + 20) },
unsafe { lane_load!(base, offs[6] as usize + 24) },
unsafe { lane_load!(base, offs[7] as usize + 28) },
];
unsafe { _mm256_loadu_si256(gathered.as_ptr().cast::<__m256i>()) }
}};
}
macro_rules! lookup8_extract {
($box_base:expr, $off:expr) => {{
let base = ($box_base).cast::<u8>();
let off = $off;
let g0 = unsafe { lane_load!(base, _mm256_extract_epi32::<0>(off) as usize) };
let g1 = unsafe { lane_load!(base, _mm256_extract_epi32::<1>(off) as usize + 4) };
let g2 = unsafe { lane_load!(base, _mm256_extract_epi32::<2>(off) as usize + 8) };
let g3 = unsafe { lane_load!(base, _mm256_extract_epi32::<3>(off) as usize + 12) };
let g4 = unsafe { lane_load!(base, _mm256_extract_epi32::<4>(off) as usize + 16) };
let g5 = unsafe { lane_load!(base, _mm256_extract_epi32::<5>(off) as usize + 20) };
let g6 = unsafe { lane_load!(base, _mm256_extract_epi32::<6>(off) as usize + 24) };
let g7 = unsafe { lane_load!(base, _mm256_extract_epi32::<7>(off) as usize + 28) };
let mut v = _mm256_setzero_si256();
v = _mm256_insert_epi32::<0>(v, g0 as i32);
v = _mm256_insert_epi32::<1>(v, g1 as i32);
v = _mm256_insert_epi32::<2>(v, g2 as i32);
v = _mm256_insert_epi32::<3>(v, g3 as i32);
v = _mm256_insert_epi32::<4>(v, g4 as i32);
v = _mm256_insert_epi32::<5>(v, g5 as i32);
v = _mm256_insert_epi32::<6>(v, g6 as i32);
_mm256_insert_epi32::<7>(v, g7 as i32)
}};
}
macro_rules! lookup4_insert {
($box_base:expr, $off:expr) => {{
let base = ($box_base).cast::<u8>();
let mut offs = [0u32; 4];
unsafe { _mm_storeu_si128(offs.as_mut_ptr().cast::<__m128i>(), $off) };
let gathered = [
unsafe { lane_load!(base, offs[0] as usize) },
unsafe { lane_load!(base, offs[1] as usize + 4) },
unsafe { lane_load!(base, offs[2] as usize + 8) },
unsafe { lane_load!(base, offs[3] as usize + 12) },
];
unsafe { _mm_loadu_si128(gathered.as_ptr().cast::<__m128i>()) }
}};
}
macro_rules! lookup4_extract {
($box_base:expr, $off:expr) => {{
let base = ($box_base).cast::<u8>();
let off = $off;
let g0 = unsafe { lane_load!(base, _mm_extract_epi32::<0>(off) as usize) };
let g1 = unsafe { lane_load!(base, _mm_extract_epi32::<1>(off) as usize + 4) };
let g2 = unsafe { lane_load!(base, _mm_extract_epi32::<2>(off) as usize + 8) };
let g3 = unsafe { lane_load!(base, _mm_extract_epi32::<3>(off) as usize + 12) };
let mut v = _mm_setzero_si128();
v = _mm_insert_epi32::<0>(v, g0 as i32);
v = _mm_insert_epi32::<1>(v, g1 as i32);
v = _mm_insert_epi32::<2>(v, g2 as i32);
_mm_insert_epi32::<3>(v, g3 as i32)
}};
}
macro_rules! f8_insert {
($s:expr, $x:expr) => {{
let s = $s;
let x = $x;
let stride_mask = _mm256_set1_epi32(0x1fe0);
let a = _mm256_and_si256(_mm256_srli_epi32::<19>(x), stride_mask);
let b = _mm256_and_si256(_mm256_srli_epi32::<11>(x), stride_mask);
let c = _mm256_and_si256(_mm256_srli_epi32::<3>(x), stride_mask);
let d = _mm256_and_si256(_mm256_slli_epi32::<5>(x), stride_mask);
let (va, vb, vc, vd) = (
lookup8_insert!(s.box_base(0), a),
lookup8_insert!(s.box_base(1), b),
lookup8_insert!(s.box_base(2), c),
lookup8_insert!(s.box_base(3), d),
);
_mm256_add_epi32(_mm256_xor_si256(_mm256_add_epi32(va, vb), vc), vd)
}};
}
macro_rules! f8_extract {
($s:expr, $x:expr) => {{
let s = $s;
let x = $x;
let stride_mask = _mm256_set1_epi32(0x1fe0);
let a = _mm256_and_si256(_mm256_srli_epi32::<19>(x), stride_mask);
let b = _mm256_and_si256(_mm256_srli_epi32::<11>(x), stride_mask);
let c = _mm256_and_si256(_mm256_srli_epi32::<3>(x), stride_mask);
let d = _mm256_and_si256(_mm256_slli_epi32::<5>(x), stride_mask);
let (va, vb, vc, vd) = (
lookup8_extract!(s.box_base(0), a),
lookup8_extract!(s.box_base(1), b),
lookup8_extract!(s.box_base(2), c),
lookup8_extract!(s.box_base(3), d),
);
_mm256_add_epi32(_mm256_xor_si256(_mm256_add_epi32(va, vb), vc), vd)
}};
}
macro_rules! f4_insert {
($s:expr, $x:expr) => {{
let s = $s;
let x = $x;
let stride_mask = _mm_set1_epi32(0xff0);
let a = _mm_and_si128(_mm_srli_epi32::<20>(x), stride_mask);
let b = _mm_and_si128(_mm_srli_epi32::<12>(x), stride_mask);
let c = _mm_and_si128(_mm_srli_epi32::<4>(x), stride_mask);
let d = _mm_and_si128(_mm_slli_epi32::<4>(x), stride_mask);
let (va, vb, vc, vd) = (
lookup4_insert!(s.box_base(0), a),
lookup4_insert!(s.box_base(1), b),
lookup4_insert!(s.box_base(2), c),
lookup4_insert!(s.box_base(3), d),
);
_mm_add_epi32(_mm_xor_si128(_mm_add_epi32(va, vb), vc), vd)
}};
}
macro_rules! f4_extract {
($s:expr, $x:expr) => {{
let s = $s;
let x = $x;
let stride_mask = _mm_set1_epi32(0xff0);
let a = _mm_and_si128(_mm_srli_epi32::<20>(x), stride_mask);
let b = _mm_and_si128(_mm_srli_epi32::<12>(x), stride_mask);
let c = _mm_and_si128(_mm_srli_epi32::<4>(x), stride_mask);
let d = _mm_and_si128(_mm_slli_epi32::<4>(x), stride_mask);
let (va, vb, vc, vd) = (
lookup4_extract!(s.box_base(0), a),
lookup4_extract!(s.box_base(1), b),
lookup4_extract!(s.box_base(2), c),
lookup4_extract!(s.box_base(3), d),
);
_mm_add_epi32(_mm_xor_si128(_mm_add_epi32(va, vb), vc), vd)
}};
}
macro_rules! f8 {
($flavor:ident, $s:expr, $x:expr) => {{
if $flavor == FLAVOR_EXTRACT {
f8_extract!($s, $x)
} else {
f8_insert!($s, $x)
}
}};
}
macro_rules! f4 {
($flavor:ident, $s:expr, $x:expr) => {{
if $flavor == FLAVOR_EXTRACT {
f4_extract!($s, $x)
} else {
f4_insert!($s, $x)
}
}};
}
macro_rules! rounds16x12 {
($state:expr, $flavor:ident, $l8:ident, $r8:ident, $l4:ident, $r4:ident) => {{
macro_rules! pair {
($i:expr) => {{
$l8 = _mm256_xor_si256($l8, $state.p8[$i]);
$l4 = _mm_xor_si128($l4, $state.p4[$i]);
$r8 = _mm256_xor_si256($r8, f8!($flavor, &$state.s8, $l8));
$r4 = _mm_xor_si128($r4, f4!($flavor, &$state.s4, $l4));
$r8 = _mm256_xor_si256($r8, $state.p8[$i + 1]);
$r4 = _mm_xor_si128($r4, $state.p4[$i + 1]);
$l8 = _mm256_xor_si256($l8, f8!($flavor, &$state.s8, $r8));
$l4 = _mm_xor_si128($l4, f4!($flavor, &$state.s4, $r4));
}};
}
pair!(0);
pair!(2);
pair!(4);
pair!(6);
pair!(8);
pair!(10);
pair!(12);
pair!(14);
$l8 = _mm256_xor_si256($l8, $state.p8[16]);
$l4 = _mm_xor_si128($l4, $state.p4[16]);
$r8 = _mm256_xor_si256($r8, $state.p8[17]);
$r4 = _mm_xor_si128($r4, $state.p4[17]);
core::mem::swap(&mut $l8, &mut $r8);
core::mem::swap(&mut $l4, &mut $r4);
}};
}
#[target_feature(enable = "avx2")]
#[inline]
fn encipher12<const FLAVOR: u8>(
state: &State12,
mut l8: __m256i,
mut r8: __m256i,
mut l4: __m128i,
mut r4: __m128i,
) -> (__m256i, __m256i, __m128i, __m128i) {
rounds16x12!(state, FLAVOR, l8, r8, l4, r4);
(l8, r8, l4, r4)
}
#[target_feature(enable = "avx2")]
#[inline]
fn expand_state_v12<const FLAVOR: u8>(
state: &mut State12,
swv8: &[__m256i; 4],
swv4: &[__m128i; 4],
kwv8: &[__m256i; 18],
kwv4: &[__m128i; 18],
) {
for ((p, &init), &k) in state.p8.iter_mut().zip(P_INIT.iter()).zip(kwv8.iter()) {
*p = _mm256_xor_si256(_mm256_set1_epi32(init as i32), k);
}
for ((p, &init), &k) in state.p4.iter_mut().zip(P_INIT.iter()).zip(kwv4.iter()) {
*p = _mm_xor_si128(_mm_set1_epi32(init as i32), k);
}
for (entry, &init) in state.s8.0.iter_mut().zip(S_INIT.as_flattened().iter()) {
*entry = [init; 8];
}
for (entry, &init) in state.s4.0.iter_mut().zip(S_INIT.as_flattened().iter()) {
*entry = [init; 4];
}
let (mut l8, mut r8) = (_mm256_setzero_si256(), _mm256_setzero_si256());
let (mut l4, mut r4) = (_mm_setzero_si128(), _mm_setzero_si128());
let mut j = 0usize;
for pair in 0..9 {
l8 = _mm256_xor_si256(l8, swv8[j % 4]);
l4 = _mm_xor_si128(l4, swv4[j % 4]);
j += 1;
r8 = _mm256_xor_si256(r8, swv8[j % 4]);
r4 = _mm_xor_si128(r4, swv4[j % 4]);
j += 1;
rounds16x12!(state, FLAVOR, l8, r8, l4, r4);
state.p8[2 * pair] = l8;
state.p4[2 * pair] = l4;
state.p8[2 * pair + 1] = r8;
state.p4[2 * pair + 1] = r4;
}
for b in 0..4 {
for pair in 0..128 {
l8 = _mm256_xor_si256(l8, swv8[j % 4]);
l4 = _mm_xor_si128(l4, swv4[j % 4]);
j += 1;
r8 = _mm256_xor_si256(r8, swv8[j % 4]);
r4 = _mm_xor_si128(r4, swv4[j % 4]);
j += 1;
rounds16x12!(state, FLAVOR, l8, r8, l4, r4);
let e = b * 256 + 2 * pair;
unsafe {
_mm256_store_si256(state.s8.0[e].as_mut_ptr().cast::<__m256i>(), l8);
_mm256_store_si256(state.s8.0[e + 1].as_mut_ptr().cast::<__m256i>(), r8);
_mm_store_si128(state.s4.0[e].as_mut_ptr().cast::<__m128i>(), l4);
_mm_store_si128(state.s4.0[e + 1].as_mut_ptr().cast::<__m128i>(), r4);
}
}
}
}
#[target_feature(enable = "avx2")]
#[inline]
fn encrypt_zero_chain_v12<const FLAVOR: u8>(state: &mut State12) {
let (mut l8, mut r8) = (_mm256_setzero_si256(), _mm256_setzero_si256());
let (mut l4, mut r4) = (_mm_setzero_si128(), _mm_setzero_si128());
for pair in 0..9 {
rounds16x12!(state, FLAVOR, l8, r8, l4, r4);
state.p8[2 * pair] = l8;
state.p4[2 * pair] = l4;
state.p8[2 * pair + 1] = r8;
state.p4[2 * pair + 1] = r4;
}
for b in 0..4 {
for pair in 0..128 {
rounds16x12!(state, FLAVOR, l8, r8, l4, r4);
let e = b * 256 + 2 * pair;
unsafe {
_mm256_store_si256(state.s8.0[e].as_mut_ptr().cast::<__m256i>(), l8);
_mm256_store_si256(state.s8.0[e + 1].as_mut_ptr().cast::<__m256i>(), r8);
_mm_store_si128(state.s4.0[e].as_mut_ptr().cast::<__m128i>(), l4);
_mm_store_si128(state.s4.0[e + 1].as_mut_ptr().cast::<__m128i>(), r4);
}
}
}
}
#[target_feature(enable = "avx2")]
#[inline]
fn expand0state_v12<const FLAVOR: u8>(
state: &mut State12,
kwv8: &[__m256i; 18],
kwv4: &[__m128i; 18],
) {
for (p, &w) in state.p8.iter_mut().zip(kwv8.iter()) {
*p = _mm256_xor_si256(*p, w);
}
for (p, &w) in state.p4.iter_mut().zip(kwv4.iter()) {
*p = _mm_xor_si128(*p, w);
}
encrypt_zero_chain_v12::<FLAVOR>(state);
}
#[target_feature(enable = "avx2")]
#[inline]
fn expand0state_salt_v12<const FLAVOR: u8>(
state: &mut State12,
swv8: &[__m256i; 4],
swv4: &[__m128i; 4],
) {
for (i, p) in state.p8.iter_mut().enumerate() {
*p = _mm256_xor_si256(*p, swv8[i & 3]);
}
for (i, p) in state.p4.iter_mut().enumerate() {
*p = _mm_xor_si128(*p, swv4[i & 3]);
}
encrypt_zero_chain_v12::<FLAVOR>(state);
}
#[target_feature(enable = "avx2")]
#[inline]
fn transpose_keys(key_words: &[[u32; 18]]) -> ([__m256i; 18], [__m128i; 18]) {
debug_assert_eq!(key_words.len(), LANES);
let mut out8 = [_mm256_setzero_si256(); 18];
for (i, v) in out8.iter_mut().enumerate() {
*v = _mm256_setr_epi32(
key_words[0][i] as i32,
key_words[1][i] as i32,
key_words[2][i] as i32,
key_words[3][i] as i32,
key_words[4][i] as i32,
key_words[5][i] as i32,
key_words[6][i] as i32,
key_words[7][i] as i32,
);
}
let mut out4 = [_mm_setzero_si128(); 18];
for (i, v) in out4.iter_mut().enumerate() {
*v = _mm_setr_epi32(
key_words[8][i] as i32,
key_words[9][i] as i32,
key_words[10][i] as i32,
key_words[11][i] as i32,
);
}
(out8, out4)
}
#[target_feature(enable = "avx2")]
#[inline]
fn transpose_salts(salt_words: &[[u32; 4]]) -> ([__m256i; 4], [__m128i; 4]) {
debug_assert_eq!(salt_words.len(), LANES);
let mut out8 = [_mm256_setzero_si256(); 4];
for (i, v) in out8.iter_mut().enumerate() {
*v = _mm256_setr_epi32(
salt_words[0][i] as i32,
salt_words[1][i] as i32,
salt_words[2][i] as i32,
salt_words[3][i] as i32,
salt_words[4][i] as i32,
salt_words[5][i] as i32,
salt_words[6][i] as i32,
salt_words[7][i] as i32,
);
}
let mut out4 = [_mm_setzero_si128(); 4];
for (i, v) in out4.iter_mut().enumerate() {
*v = _mm_setr_epi32(
salt_words[8][i] as i32,
salt_words[9][i] as i32,
salt_words[10][i] as i32,
salt_words[11][i] as i32,
);
}
(out8, out4)
}
#[target_feature(enable = "avx2")]
#[inline]
unsafe fn bcrypt12<const FLAVOR: u8>(
cost: u32,
key_words: &[[u32; 18]],
salt_words: &[[u32; 4]],
outs: &mut [[u8; 24]],
) {
debug_assert!((4..=31).contains(&cost));
debug_assert_eq!(key_words.len(), LANES);
debug_assert_eq!(salt_words.len(), LANES);
debug_assert_eq!(outs.len(), LANES);
#[cfg_attr(not(feature = "zeroize"), allow(unused_mut))]
let (mut kwv8, mut kwv4) = transpose_keys(key_words);
let (swv8, swv4) = transpose_salts(salt_words);
let mut state = State12 {
p8: [_mm256_setzero_si256(); 18],
p4: [_mm_setzero_si128(); 18],
s8: SBoxes8([[0; 8]; 1024]),
s4: SBoxes4([[0; 4]; 1024]),
};
expand_state_v12::<FLAVOR>(&mut state, &swv8, &swv4, &kwv8, &kwv4);
for _ in 0..(1u64 << cost) {
expand0state_v12::<FLAVOR>(&mut state, &kwv8, &kwv4);
expand0state_salt_v12::<FLAVOR>(&mut state, &swv8, &swv4);
}
let mut cdata8 = [
_mm256_set1_epi32(0x4f72_7068),
_mm256_set1_epi32(0x6561_6e42),
_mm256_set1_epi32(0x6568_6f6c),
_mm256_set1_epi32(0x6465_7253),
_mm256_set1_epi32(0x6372_7944),
_mm256_set1_epi32(0x6f75_6274),
];
let mut cdata4 = [
_mm_set1_epi32(0x4f72_7068),
_mm_set1_epi32(0x6561_6e42),
_mm_set1_epi32(0x6568_6f6c),
_mm_set1_epi32(0x6465_7253),
_mm_set1_epi32(0x6372_7944),
_mm_set1_epi32(0x6f75_6274),
];
for _ in 0..64 {
for pair in 0..3 {
let (l8, r8, l4, r4) = encipher12::<FLAVOR>(
&state,
cdata8[2 * pair],
cdata8[2 * pair + 1],
cdata4[2 * pair],
cdata4[2 * pair + 1],
);
cdata8[2 * pair] = l8;
cdata8[2 * pair + 1] = r8;
cdata4[2 * pair] = l4;
cdata4[2 * pair + 1] = r4;
}
}
for (w, &cv) in cdata8.iter().enumerate() {
let mut words = [0u32; 8];
unsafe { _mm256_storeu_si256(words.as_mut_ptr().cast::<__m256i>(), cv) };
for (lane, out) in outs[..8].iter_mut().enumerate() {
out[4 * w..4 * w + 4].copy_from_slice(&words[lane].to_be_bytes());
}
}
for (w, &cv) in cdata4.iter().enumerate() {
let mut words = [0u32; 4];
unsafe { _mm_storeu_si128(words.as_mut_ptr().cast::<__m128i>(), cv) };
for (lane, out) in outs[8..].iter_mut().enumerate() {
out[4 * w..4 * w + 4].copy_from_slice(&words[lane].to_be_bytes());
}
}
#[cfg(feature = "zeroize")]
{
crate::wipe::secure_wipe_u32(unsafe {
core::slice::from_raw_parts_mut(state.p8.as_mut_ptr().cast::<u32>(), 18 * 8)
});
crate::wipe::secure_wipe_u32(unsafe {
core::slice::from_raw_parts_mut(state.p4.as_mut_ptr().cast::<u32>(), 18 * 4)
});
crate::wipe::secure_wipe_u32(state.s8.0.as_flattened_mut());
crate::wipe::secure_wipe_u32(state.s4.0.as_flattened_mut());
crate::wipe::secure_wipe_u32(unsafe {
core::slice::from_raw_parts_mut(cdata8.as_mut_ptr().cast::<u32>(), 6 * 8)
});
crate::wipe::secure_wipe_u32(unsafe {
core::slice::from_raw_parts_mut(cdata4.as_mut_ptr().cast::<u32>(), 6 * 4)
});
crate::wipe::secure_wipe_u32(unsafe {
core::slice::from_raw_parts_mut(kwv8.as_mut_ptr().cast::<u32>(), 18 * 8)
});
crate::wipe::secure_wipe_u32(unsafe {
core::slice::from_raw_parts_mut(kwv4.as_mut_ptr().cast::<u32>(), 18 * 4)
});
}
}
#[derive(Copy, Clone)]
enum Flavor {
Insert,
Extract,
}
const FLAVOR_UNINIT: u8 = 0;
const FLAVOR_INSERT: u8 = 2;
const FLAVOR_EXTRACT: u8 = 3;
static CACHED_FLAVOR: AtomicU8 = AtomicU8::new(FLAVOR_UNINIT);
const fn flavor_to_u8(flavor: Flavor) -> u8 {
match flavor {
Flavor::Insert => FLAVOR_INSERT,
Flavor::Extract => FLAVOR_EXTRACT,
}
}
const fn flavor_from_u8(value: u8) -> Flavor {
match value {
FLAVOR_EXTRACT => Flavor::Extract,
_ => Flavor::Insert,
}
}
#[inline]
fn flavor() -> Flavor {
let cached = CACHED_FLAVOR.load(Ordering::Relaxed);
if cached == FLAVOR_UNINIT {
pick_and_cache()
} else {
flavor_from_u8(cached)
}
}
#[cold]
#[inline(never)]
fn pick_and_cache() -> Flavor {
let picked = pick_flavor();
CACHED_FLAVOR.store(flavor_to_u8(picked), Ordering::Relaxed);
picked
}
#[cfg(all(feature = "std", not(debug_assertions), not(miri)))]
fn pick_flavor() -> Flavor {
shootout()
}
#[cfg(not(all(feature = "std", not(debug_assertions), not(miri))))]
fn pick_flavor() -> Flavor {
Flavor::Insert
}
#[cfg(all(feature = "std", not(debug_assertions), not(miri)))]
fn shootout() -> Flavor {
use std::time::Instant;
let mut rng = 0xBC79_7A5A_F1A0_0004u64;
let mut next_u32 = move || {
rng = rng.wrapping_add(0x9E37_79B9_7F4A_7C15);
let mut z = rng;
z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9);
z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB);
(z ^ (z >> 31)) as u32
};
let mut kws = [[0u32; 18]; LANES];
let mut sws = [[0u32; 4]; LANES];
for w in kws.as_flattened_mut().iter_mut().chain(sws.as_flattened_mut()) {
*w = next_u32();
}
let mut outs = [[0u8; 24]; LANES];
unsafe { bcrypt12::<FLAVOR_INSERT>(4, &kws, &sws, &mut outs) };
let reference = outs;
unsafe { bcrypt12::<FLAVOR_EXTRACT>(4, &kws, &sws, &mut outs) };
assert_eq!(
outs, reference,
"avx2x12 flavor shootout: insert and extract kernels diverged — a kernel bug, not timing"
);
let (mut best_insert, mut best_extract) = (f64::MAX, f64::MAX);
for _ in 0..3 {
let start = Instant::now();
unsafe { bcrypt12::<FLAVOR_INSERT>(4, &kws, &sws, &mut outs) };
best_insert = best_insert.min(start.elapsed().as_secs_f64());
let start = Instant::now();
unsafe { bcrypt12::<FLAVOR_EXTRACT>(4, &kws, &sws, &mut outs) };
best_extract = best_extract.min(start.elapsed().as_secs_f64());
core::hint::black_box(&mut outs);
}
if best_insert <= best_extract {
Flavor::Insert
} else {
Flavor::Extract
}
}
#[target_feature(enable = "avx2")]
#[inline]
unsafe fn bcrypt_lanes_impl(
cost: u32,
key_words: &[[u32; 18]],
salt_words: &[[u32; 4]],
outs: &mut [[u8; 24]],
) {
debug_assert!((4..=31).contains(&cost));
debug_assert_eq!(key_words.len(), LANES);
debug_assert_eq!(salt_words.len(), LANES);
debug_assert_eq!(outs.len(), LANES);
match flavor() {
Flavor::Insert => unsafe { bcrypt12::<FLAVOR_INSERT>(cost, key_words, salt_words, outs) },
Flavor::Extract => unsafe { bcrypt12::<FLAVOR_EXTRACT>(cost, key_words, salt_words, outs) },
}
}
#[target_feature(enable = "avx2")]
pub unsafe fn bcrypt_lanes(
cost: u32,
key_words: &[[u32; 18]],
salt_words: &[[u32; 4]],
outs: &mut [[u8; 24]],
) {
debug_assert!((4..=31).contains(&cost));
debug_assert_eq!(key_words.len(), LANES);
debug_assert_eq!(salt_words.len(), LANES);
debug_assert_eq!(outs.len(), LANES);
unsafe { bcrypt_lanes_impl(cost, key_words, salt_words, outs) }
}
#[cfg(test)]
mod tests {
use super::*;
use crate::base64;
use crate::eks::{Backend, expand_key_words, salt_words};
fn key_words_for(password: &[u8]) -> [u32; 18] {
let mut key = [0u8; 72];
let copied = password.len().min(72);
key[..copied].copy_from_slice(&password[..copied]);
expand_key_words(&key, (copied + 1).min(72))
}
fn avx2x12_available() -> bool {
!cfg!(miri) && Backend::Avx2x12.is_available()
}
fn fixed_words() -> ([[u32; 18]; LANES], [[u32; 4]; LANES]) {
let mut rng = 0x853C_49E6_748F_EA9Cu64; let mut next_u32 = move || {
rng = rng.wrapping_add(0x9E37_79B9_7F4A_7C15);
let mut z = rng;
z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9);
z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB);
(z ^ (z >> 31)) as u32
};
let mut kws = [[0u32; 18]; LANES];
let mut sws = [[0u32; 4]; LANES];
for w in kws.as_flattened_mut().iter_mut().chain(sws.as_flattened_mut()) {
*w = next_u32();
}
(kws, sws)
}
#[test]
fn flavors_agree() {
if !avx2x12_available() {
return;
}
let (kws, sws) = fixed_words();
let (mut inserted, mut extracted) = ([[0u8; 24]; LANES], [[0u8; 24]; LANES]);
unsafe {
bcrypt12::<FLAVOR_INSERT>(4, &kws, &sws, &mut inserted);
bcrypt12::<FLAVOR_EXTRACT>(4, &kws, &sws, &mut extracted);
}
assert_eq!(inserted, extracted, "insert and extract flavors diverged");
}
#[test]
fn lanes_match_scalar() {
if !avx2x12_available() {
return;
}
let vector_salt =
base64::decode_16(b"CCCCCCCCCCCCCCCCCCCCC.").expect("test salt must decode");
let passwords: [&[u8]; 12] = [
b"",
b"U*U",
b"hunter2",
&[0xAA; 72],
b"correct horse",
&[0x01],
&[0x55; 71],
&[0xFF; 100],
b"lane eight",
&[0x5A; 72],
&[0xFE; 2],
b"0123456789",
];
let kws: [[u32; 18]; 12] = core::array::from_fn(|i| key_words_for(passwords[i]));
let sws: [[u32; 4]; 12] = [
salt_words(&[0x11; 16]),
salt_words(&vector_salt),
salt_words(&[0x33; 16]),
salt_words(&[0x44; 16]),
salt_words(&[0x55; 16]),
salt_words(&[0x66; 16]),
salt_words(&[0x77; 16]),
salt_words(&[0x88; 16]),
salt_words(&[0x99; 16]),
salt_words(&[0xAA; 16]),
salt_words(&[0xBB; 16]),
salt_words(&[0xCC; 16]),
];
let mut outs = [[0u8; 24]; 12];
unsafe { bcrypt_lanes(5, &kws, &sws, &mut outs) };
for lane in 0..LANES {
let mut expected = [0u8; 24];
crate::eks::scalar::bcrypt_lanes(
5,
&kws[lane..lane + 1],
&sws[lane..lane + 1],
core::slice::from_mut(&mut expected),
);
assert_eq!(outs[lane], expected, "lane {lane} diverged from scalar");
}
}
#[test]
fn flavor_pick_is_cached() {
if !avx2x12_available() {
return;
}
let first = flavor();
assert_ne!(CACHED_FLAVOR.load(Ordering::Relaxed), FLAVOR_UNINIT);
assert_eq!(flavor_to_u8(first), CACHED_FLAVOR.load(Ordering::Relaxed));
}
}