#[cfg(target_arch = "x86")]
use core::arch::x86::{
__m128i, _mm_add_epi32, _mm_and_si128, _mm_extract_epi32, _mm_insert_epi32,
_mm_loadu_si128, _mm_set1_epi32, _mm_setr_epi32, _mm_setzero_si128, _mm_srli_epi32,
_mm_store_si128, _mm_storeu_si128, _mm_xor_si128,
};
#[cfg(target_arch = "x86_64")]
use core::arch::x86_64::{
__m128i, _mm_add_epi32, _mm_and_si128, _mm_extract_epi32, _mm_insert_epi32,
_mm_loadu_si128, _mm_set1_epi32, _mm_setr_epi32, _mm_setzero_si128, _mm_srli_epi32,
_mm_store_si128, _mm_storeu_si128, _mm_xor_si128,
};
use crate::consts::{P_INIT, S_INIT};
const LANES: usize = 4;
#[repr(C, align(64))]
struct SBoxes4x([[u32; 4]; 1024]);
impl SBoxes4x {
#[inline(always)]
fn box_base(&self, b: usize) -> *const u32 {
debug_assert!(b < 4);
unsafe { self.0.as_ptr().cast::<u32>().add(b * 256 * 4) }
}
}
struct State4x {
p: [__m128i; 18],
s: SBoxes4x,
}
#[target_feature(enable = "sse4.1")]
#[inline]
unsafe fn lookup4(box_base: *const u32, idx: __m128i) -> __m128i {
unsafe { lookup4_stack(box_base, idx) }
}
#[target_feature(enable = "sse4.1")]
#[inline]
unsafe fn lookup4_stack(box_base: *const u32, idx: __m128i) -> __m128i {
let mut ix = [0u32; 4];
unsafe { _mm_storeu_si128(ix.as_mut_ptr().cast::<__m128i>(), idx) };
let gathered = [
unsafe { *box_base.add(ix[0] as usize * 4) },
unsafe { *box_base.add(ix[1] as usize * 4 + 1) },
unsafe { *box_base.add(ix[2] as usize * 4 + 2) },
unsafe { *box_base.add(ix[3] as usize * 4 + 3) },
];
unsafe { _mm_loadu_si128(gathered.as_ptr().cast::<__m128i>()) }
}
#[allow(dead_code)] #[target_feature(enable = "sse4.1")]
#[inline]
unsafe fn lookup4_pinsr(box_base: *const u32, idx: __m128i) -> __m128i {
unsafe {
let v = _mm_insert_epi32::<0>(
_mm_setzero_si128(),
*box_base.add(_mm_extract_epi32::<0>(idx) as u32 as usize * 4) as i32,
);
let v = _mm_insert_epi32::<1>(
v,
*box_base.add(_mm_extract_epi32::<1>(idx) as u32 as usize * 4 + 1) as i32,
);
let v = _mm_insert_epi32::<2>(
v,
*box_base.add(_mm_extract_epi32::<2>(idx) as u32 as usize * 4 + 2) as i32,
);
_mm_insert_epi32::<3>(
v,
*box_base.add(_mm_extract_epi32::<3>(idx) as u32 as usize * 4 + 3) as i32,
)
}
}
#[target_feature(enable = "sse4.1")]
#[inline]
fn f4(s: &SBoxes4x, x: __m128i) -> __m128i {
let byte_mask = _mm_set1_epi32(0xff);
let a = _mm_srli_epi32::<24>(x);
let b = _mm_and_si128(_mm_srli_epi32::<16>(x), byte_mask);
let c = _mm_and_si128(_mm_srli_epi32::<8>(x), byte_mask);
let d = _mm_and_si128(x, byte_mask);
let (va, vb, vc, vd) = unsafe {
(
lookup4(s.box_base(0), a),
lookup4(s.box_base(1), b),
lookup4(s.box_base(2), c),
lookup4(s.box_base(3), d),
)
};
_mm_add_epi32(_mm_xor_si128(_mm_add_epi32(va, vb), vc), vd)
}
macro_rules! rounds16 {
($state:expr, $l:ident, $r:ident) => {{
macro_rules! pair {
($i:expr) => {{
$l = _mm_xor_si128($l, $state.p[$i]);
$r = _mm_xor_si128($r, f4(&$state.s, $l));
$r = _mm_xor_si128($r, $state.p[$i + 1]);
$l = _mm_xor_si128($l, f4(&$state.s, $r));
}};
}
pair!(0);
pair!(2);
pair!(4);
pair!(6);
pair!(8);
pair!(10);
pair!(12);
pair!(14);
$l = _mm_xor_si128($l, $state.p[16]);
$r = _mm_xor_si128($r, $state.p[17]);
core::mem::swap(&mut $l, &mut $r);
}};
}
#[target_feature(enable = "sse4.1")]
#[inline]
fn encipher4(state: &State4x, mut l: __m128i, mut r: __m128i) -> (__m128i, __m128i) {
rounds16!(state, l, r);
(l, r)
}
#[target_feature(enable = "sse4.1")]
#[inline]
fn expand_state_v4(state: &mut State4x, swv: &[__m128i; 4], kwv: &[__m128i; 18]) {
for ((p, &init), &k) in state.p.iter_mut().zip(P_INIT.iter()).zip(kwv.iter()) {
*p = _mm_xor_si128(_mm_set1_epi32(init as i32), k);
}
for (entry, &init) in state.s.0.iter_mut().zip(S_INIT.as_flattened().iter()) {
*entry = [init; 4];
}
let (mut l, mut r) = (_mm_setzero_si128(), _mm_setzero_si128());
let mut j = 0usize;
for pair in 0..9 {
l = _mm_xor_si128(l, swv[j % 4]);
j += 1;
r = _mm_xor_si128(r, swv[j % 4]);
j += 1;
rounds16!(state, l, r);
state.p[2 * pair] = l;
state.p[2 * pair + 1] = r;
}
for b in 0..4 {
for pair in 0..128 {
l = _mm_xor_si128(l, swv[j % 4]);
j += 1;
r = _mm_xor_si128(r, swv[j % 4]);
j += 1;
rounds16!(state, l, r);
let e = b * 256 + 2 * pair;
unsafe {
_mm_store_si128(state.s.0[e].as_mut_ptr().cast::<__m128i>(), l);
_mm_store_si128(state.s.0[e + 1].as_mut_ptr().cast::<__m128i>(), r);
}
}
}
}
#[target_feature(enable = "sse4.1")]
#[inline]
fn encrypt_zero_chain_v4(state: &mut State4x) {
let (mut l, mut r) = (_mm_setzero_si128(), _mm_setzero_si128());
for pair in 0..9 {
rounds16!(state, l, r);
state.p[2 * pair] = l;
state.p[2 * pair + 1] = r;
}
for b in 0..4 {
for pair in 0..128 {
rounds16!(state, l, r);
let e = b * 256 + 2 * pair;
unsafe {
_mm_store_si128(state.s.0[e].as_mut_ptr().cast::<__m128i>(), l);
_mm_store_si128(state.s.0[e + 1].as_mut_ptr().cast::<__m128i>(), r);
}
}
}
}
#[target_feature(enable = "sse4.1")]
#[inline]
fn expand0state_v4(state: &mut State4x, wv: &[__m128i; 18]) {
for (p, &w) in state.p.iter_mut().zip(wv.iter()) {
*p = _mm_xor_si128(*p, w);
}
encrypt_zero_chain_v4(state);
}
#[target_feature(enable = "sse4.1")]
#[inline]
fn expand0state_salt_v4(state: &mut State4x, swv: &[__m128i; 4]) {
for (i, p) in state.p.iter_mut().enumerate() {
*p = _mm_xor_si128(*p, swv[i & 3]);
}
encrypt_zero_chain_v4(state);
}
#[target_feature(enable = "sse4.1")]
#[inline]
fn transpose_keys(key_words: &[[u32; 18]]) -> [__m128i; 18] {
debug_assert_eq!(key_words.len(), LANES);
let mut out = [_mm_setzero_si128(); 18];
for (i, v) in out.iter_mut().enumerate() {
*v = _mm_setr_epi32(
key_words[0][i] as i32,
key_words[1][i] as i32,
key_words[2][i] as i32,
key_words[3][i] as i32,
);
}
out
}
#[target_feature(enable = "sse4.1")]
#[inline]
fn transpose_salts(salt_words: &[[u32; 4]]) -> [__m128i; 4] {
debug_assert_eq!(salt_words.len(), LANES);
let mut out = [_mm_setzero_si128(); 4];
for (i, v) in out.iter_mut().enumerate() {
*v = _mm_setr_epi32(
salt_words[0][i] as i32,
salt_words[1][i] as i32,
salt_words[2][i] as i32,
salt_words[3][i] as i32,
);
}
out
}
#[target_feature(enable = "sse4.1")]
#[inline]
unsafe fn bcrypt_lanes_impl(
cost: u32,
key_words: &[[u32; 18]],
salt_words: &[[u32; 4]],
outs: &mut [[u8; 24]],
) {
debug_assert!((4..=31).contains(&cost));
debug_assert_eq!(key_words.len(), LANES);
debug_assert_eq!(salt_words.len(), LANES);
debug_assert_eq!(outs.len(), LANES);
#[cfg_attr(not(feature = "zeroize"), allow(unused_mut))]
let mut kwv = transpose_keys(key_words);
let swv = transpose_salts(salt_words);
let mut state = State4x {
p: [_mm_setzero_si128(); 18],
s: SBoxes4x([[0; 4]; 1024]),
};
expand_state_v4(&mut state, &swv, &kwv);
for _ in 0..(1u64 << cost) {
expand0state_v4(&mut state, &kwv);
expand0state_salt_v4(&mut state, &swv);
}
let mut cdata = [
_mm_set1_epi32(0x4f72_7068),
_mm_set1_epi32(0x6561_6e42),
_mm_set1_epi32(0x6568_6f6c),
_mm_set1_epi32(0x6465_7253),
_mm_set1_epi32(0x6372_7944),
_mm_set1_epi32(0x6f75_6274),
];
for _ in 0..64 {
for pair in 0..3 {
let (l, r) = encipher4(&state, cdata[2 * pair], cdata[2 * pair + 1]);
cdata[2 * pair] = l;
cdata[2 * pair + 1] = r;
}
}
for (w, &cv) in cdata.iter().enumerate() {
let mut words = [0u32; 4];
unsafe { _mm_storeu_si128(words.as_mut_ptr().cast::<__m128i>(), cv) };
for (lane, out) in outs.iter_mut().enumerate() {
out[4 * w..4 * w + 4].copy_from_slice(&words[lane].to_be_bytes());
}
}
#[cfg(feature = "zeroize")]
{
crate::wipe::secure_wipe_u32(unsafe {
core::slice::from_raw_parts_mut(state.p.as_mut_ptr().cast::<u32>(), 18 * 4)
});
crate::wipe::secure_wipe_u32(state.s.0.as_flattened_mut());
crate::wipe::secure_wipe_u32(unsafe {
core::slice::from_raw_parts_mut(cdata.as_mut_ptr().cast::<u32>(), 6 * 4)
});
crate::wipe::secure_wipe_u32(unsafe {
core::slice::from_raw_parts_mut(kwv.as_mut_ptr().cast::<u32>(), 18 * 4)
});
}
}
#[target_feature(enable = "sse4.1")]
pub unsafe fn bcrypt_lanes(
cost: u32,
key_words: &[[u32; 18]],
salt_words: &[[u32; 4]],
outs: &mut [[u8; 24]],
) {
debug_assert!((4..=31).contains(&cost));
debug_assert_eq!(key_words.len(), LANES);
debug_assert_eq!(salt_words.len(), LANES);
debug_assert_eq!(outs.len(), LANES);
unsafe { bcrypt_lanes_impl(cost, key_words, salt_words, outs) }
}
#[cfg(test)]
mod tests {
use super::*;
use crate::base64;
use crate::eks::{Backend, expand_key_words, salt_words};
fn key_words_for(password: &[u8]) -> [u32; 18] {
let mut key = [0u8; 72];
let copied = password.len().min(72);
key[..copied].copy_from_slice(&password[..copied]);
expand_key_words(&key, (copied + 1).min(72))
}
fn sse41_available() -> bool {
!cfg!(miri) && Backend::Sse41.is_available()
}
#[test]
fn lookup_spellings_agree() {
if !sse41_available() {
return;
}
let mut boxes = SBoxes4x([[0; 4]; 1024]);
for (entry, words) in boxes.0.iter_mut().enumerate() {
*words = [
entry as u32,
(entry as u32) ^ 0xAAAA_AAAA,
(entry as u32).wrapping_mul(31),
!(entry as u32),
];
}
for i in 0..256u32 {
let idx = unsafe {
_mm_setr_epi32(i as i32, (255 - i) as i32, (i ^ 0x5A) as i32, ((i * 7) & 0xFF) as i32)
};
let (a, b) = unsafe {
(
lookup4_stack(boxes.box_base((i % 4) as usize), idx),
lookup4_pinsr(boxes.box_base((i % 4) as usize), idx),
)
};
let (mut wa, mut wb) = ([0u32; 4], [0u32; 4]);
unsafe {
_mm_storeu_si128(wa.as_mut_ptr().cast::<__m128i>(), a);
_mm_storeu_si128(wb.as_mut_ptr().cast::<__m128i>(), b);
}
assert_eq!(wa, wb, "spellings diverged at index set {i}");
}
}
#[test]
fn lanes_match_scalar() {
if !sse41_available() {
return;
}
let vector_salt =
base64::decode_16(b"CCCCCCCCCCCCCCCCCCCCC.").expect("test salt must decode");
let passwords: [&[u8]; 4] = [b"", b"U*U", b"hunter2", &[0xAA; 72]];
let kws: [[u32; 18]; 4] = core::array::from_fn(|i| key_words_for(passwords[i]));
let sws: [[u32; 4]; 4] = [
salt_words(&[0x11; 16]),
salt_words(&vector_salt),
salt_words(&[0x33; 16]),
salt_words(&[0x44; 16]),
];
let mut outs = [[0u8; 24]; 4];
unsafe { bcrypt_lanes(5, &kws, &sws, &mut outs) };
for lane in 0..LANES {
let mut expected = [0u8; 24];
crate::eks::scalar::bcrypt_lanes(
5,
&kws[lane..lane + 1],
&sws[lane..lane + 1],
core::slice::from_mut(&mut expected),
);
assert_eq!(outs[lane], expected, "lane {lane} diverged from scalar");
}
}
}