use core::arch::wasm32::{
i32x4_add, i32x4_shl, u32x4_shr, u32x4_splat, v128, v128_and, v128_load, v128_store, v128_xor,
};
use crate::consts::{P_INIT, S_INIT};
static LANE_OFFSETS_A: [u32; 4] = [0, 1, 2, 3];
static LANE_OFFSETS_B: [u32; 4] = [4, 5, 6, 7];
pub(crate) const LANES: usize = 8;
#[repr(C, align(64))]
struct SBoxes8x([[u32; 8]; 1024]);
impl SBoxes8x {
#[inline(always)]
fn box_base(&self, b: usize) -> *const u32 {
debug_assert!(b < 4);
unsafe { self.0.as_ptr().cast::<u32>().add(b * 256 * 8) }
}
}
struct State8x {
pa: [v128; 18],
pb: [v128; 18],
s: SBoxes8x,
}
#[target_feature(enable = "simd128")]
#[inline]
unsafe fn lookup8(box_base: *const u32, sa: v128, sb: v128) -> (v128, v128) {
let mut ix = [0u32; 8];
unsafe {
v128_store(ix.as_mut_ptr().cast::<v128>(), sa);
v128_store(ix.as_mut_ptr().add(4).cast::<v128>(), sb);
}
let a0 = unsafe { *box_base.add(ix[0] as usize) };
let b0 = unsafe { *box_base.add(ix[4] as usize) };
let a1 = unsafe { *box_base.add(ix[1] as usize) };
let b1 = unsafe { *box_base.add(ix[5] as usize) };
let a2 = unsafe { *box_base.add(ix[2] as usize) };
let b2 = unsafe { *box_base.add(ix[6] as usize) };
let a3 = unsafe { *box_base.add(ix[3] as usize) };
let b3 = unsafe { *box_base.add(ix[7] as usize) };
let mut gathered = [0u32; 8];
unsafe {
core::ptr::write_volatile(&mut gathered[0], a0);
core::ptr::write_volatile(&mut gathered[1], a1);
core::ptr::write_volatile(&mut gathered[2], a2);
core::ptr::write_volatile(&mut gathered[3], a3);
core::ptr::write_volatile(&mut gathered[4], b0);
core::ptr::write_volatile(&mut gathered[5], b1);
core::ptr::write_volatile(&mut gathered[6], b2);
core::ptr::write_volatile(&mut gathered[7], b3);
(
core::ptr::read_volatile(gathered.as_ptr().cast::<v128>()),
core::ptr::read_volatile(gathered.as_ptr().add(4).cast::<v128>()),
)
}
}
#[target_feature(enable = "simd128")]
#[inline]
fn f8(s: &SBoxes8x, xa: v128, xb: v128) -> (v128, v128) {
let byte_mask = u32x4_splat(0xff);
let (off_a, off_b) = unsafe {
(
v128_load(LANE_OFFSETS_A.as_ptr().cast::<v128>()),
v128_load(LANE_OFFSETS_B.as_ptr().cast::<v128>()),
)
};
let aa = i32x4_add(i32x4_shl(u32x4_shr(xa, 24), 3), off_a);
let ab = i32x4_add(i32x4_shl(u32x4_shr(xb, 24), 3), off_b);
let ba = i32x4_add(i32x4_shl(v128_and(u32x4_shr(xa, 16), byte_mask), 3), off_a);
let bb = i32x4_add(i32x4_shl(v128_and(u32x4_shr(xb, 16), byte_mask), 3), off_b);
let ca = i32x4_add(i32x4_shl(v128_and(u32x4_shr(xa, 8), byte_mask), 3), off_a);
let cb = i32x4_add(i32x4_shl(v128_and(u32x4_shr(xb, 8), byte_mask), 3), off_b);
let da = i32x4_add(i32x4_shl(v128_and(xa, byte_mask), 3), off_a);
let db = i32x4_add(i32x4_shl(v128_and(xb, byte_mask), 3), off_b);
let (vaa, vab) = unsafe { lookup8(s.box_base(0), aa, ab) };
let (vba, vbb) = unsafe { lookup8(s.box_base(1), ba, bb) };
let (vca, vcb) = unsafe { lookup8(s.box_base(2), ca, cb) };
let (vda, vdb) = unsafe { lookup8(s.box_base(3), da, db) };
(
i32x4_add(v128_xor(i32x4_add(vaa, vba), vca), vda),
i32x4_add(v128_xor(i32x4_add(vab, vbb), vcb), vdb),
)
}
macro_rules! rounds16x2 {
($state:expr, $la:ident, $ra:ident, $lb:ident, $rb:ident) => {{
macro_rules! pair {
($i:expr) => {{
$la = v128_xor($la, $state.pa[$i]);
$lb = v128_xor($lb, $state.pb[$i]);
let (fa, fb) = f8(&$state.s, $la, $lb);
$ra = v128_xor($ra, fa);
$rb = v128_xor($rb, fb);
$ra = v128_xor($ra, $state.pa[$i + 1]);
$rb = v128_xor($rb, $state.pb[$i + 1]);
let (fa, fb) = f8(&$state.s, $ra, $rb);
$la = v128_xor($la, fa);
$lb = v128_xor($lb, fb);
}};
}
pair!(0);
pair!(2);
pair!(4);
pair!(6);
pair!(8);
pair!(10);
pair!(12);
pair!(14);
$la = v128_xor($la, $state.pa[16]);
$lb = v128_xor($lb, $state.pb[16]);
$ra = v128_xor($ra, $state.pa[17]);
$rb = v128_xor($rb, $state.pb[17]);
core::mem::swap(&mut $la, &mut $ra);
core::mem::swap(&mut $lb, &mut $rb);
}};
}
#[target_feature(enable = "simd128")]
#[inline]
fn encipher8(
state: &State8x,
mut la: v128,
mut ra: v128,
mut lb: v128,
mut rb: v128,
) -> (v128, v128, v128, v128) {
rounds16x2!(state, la, ra, lb, rb);
(la, ra, lb, rb)
}
#[target_feature(enable = "simd128")]
#[inline]
fn expand_state_v8(
state: &mut State8x,
swa: &[v128; 4],
swb: &[v128; 4],
kwa: &[v128; 18],
kwb: &[v128; 18],
) {
for (i, &init) in P_INIT.iter().enumerate() {
state.pa[i] = v128_xor(u32x4_splat(init), kwa[i]);
state.pb[i] = v128_xor(u32x4_splat(init), kwb[i]);
}
for (entry, &init) in state.s.0.iter_mut().zip(S_INIT.as_flattened().iter()) {
*entry = [init; 8];
}
let zero = u32x4_splat(0);
let (mut la, mut ra, mut lb, mut rb) = (zero, zero, zero, zero);
let mut j = 0usize;
for pair in 0..9 {
la = v128_xor(la, swa[j % 4]);
lb = v128_xor(lb, swb[j % 4]);
j += 1;
ra = v128_xor(ra, swa[j % 4]);
rb = v128_xor(rb, swb[j % 4]);
j += 1;
rounds16x2!(state, la, ra, lb, rb);
state.pa[2 * pair] = la;
state.pa[2 * pair + 1] = ra;
state.pb[2 * pair] = lb;
state.pb[2 * pair + 1] = rb;
}
for b in 0..4 {
for pair in 0..128 {
la = v128_xor(la, swa[j % 4]);
lb = v128_xor(lb, swb[j % 4]);
j += 1;
ra = v128_xor(ra, swa[j % 4]);
rb = v128_xor(rb, swb[j % 4]);
j += 1;
rounds16x2!(state, la, ra, lb, rb);
let e = b * 256 + 2 * pair;
unsafe {
let e0 = state.s.0[e].as_mut_ptr();
v128_store(e0.cast::<v128>(), la);
v128_store(e0.add(4).cast::<v128>(), lb);
let e1 = state.s.0[e + 1].as_mut_ptr();
v128_store(e1.cast::<v128>(), ra);
v128_store(e1.add(4).cast::<v128>(), rb);
}
}
}
}
#[target_feature(enable = "simd128")]
#[inline]
fn encrypt_zero_chain_v8(state: &mut State8x) {
let zero = u32x4_splat(0);
let (mut la, mut ra, mut lb, mut rb) = (zero, zero, zero, zero);
for pair in 0..9 {
rounds16x2!(state, la, ra, lb, rb);
state.pa[2 * pair] = la;
state.pa[2 * pair + 1] = ra;
state.pb[2 * pair] = lb;
state.pb[2 * pair + 1] = rb;
}
for b in 0..4 {
for pair in 0..128 {
rounds16x2!(state, la, ra, lb, rb);
let e = b * 256 + 2 * pair;
unsafe {
let e0 = state.s.0[e].as_mut_ptr();
v128_store(e0.cast::<v128>(), la);
v128_store(e0.add(4).cast::<v128>(), lb);
let e1 = state.s.0[e + 1].as_mut_ptr();
v128_store(e1.cast::<v128>(), ra);
v128_store(e1.add(4).cast::<v128>(), rb);
}
}
}
}
#[target_feature(enable = "simd128")]
#[inline]
fn expand0state_v8(state: &mut State8x, kwa: &[v128; 18], kwb: &[v128; 18]) {
for ((pa, pb), (&ka, &kb)) in state
.pa
.iter_mut()
.zip(state.pb.iter_mut())
.zip(kwa.iter().zip(kwb.iter()))
{
*pa = v128_xor(*pa, ka);
*pb = v128_xor(*pb, kb);
}
encrypt_zero_chain_v8(state);
}
#[target_feature(enable = "simd128")]
#[inline]
fn expand0state_salt_v8(state: &mut State8x, swa: &[v128; 4], swb: &[v128; 4]) {
for (i, (pa, pb)) in state.pa.iter_mut().zip(state.pb.iter_mut()).enumerate() {
*pa = v128_xor(*pa, swa[i & 3]);
*pb = v128_xor(*pb, swb[i & 3]);
}
encrypt_zero_chain_v8(state);
}
#[target_feature(enable = "simd128")]
#[inline]
fn transpose_keys(key_words: &[[u32; 18]]) -> ([v128; 18], [v128; 18]) {
debug_assert_eq!(key_words.len(), LANES);
let mut a = [u32x4_splat(0); 18];
let mut b = [u32x4_splat(0); 18];
for (i, (va, vb)) in a.iter_mut().zip(b.iter_mut()).enumerate() {
let wa = [
key_words[0][i],
key_words[1][i],
key_words[2][i],
key_words[3][i],
];
let wb = [
key_words[4][i],
key_words[5][i],
key_words[6][i],
key_words[7][i],
];
unsafe {
*va = v128_load(wa.as_ptr().cast::<v128>());
*vb = v128_load(wb.as_ptr().cast::<v128>());
}
}
(a, b)
}
#[target_feature(enable = "simd128")]
#[inline]
fn transpose_salts(salt_words: &[[u32; 4]]) -> ([v128; 4], [v128; 4]) {
debug_assert_eq!(salt_words.len(), LANES);
let mut a = [u32x4_splat(0); 4];
let mut b = [u32x4_splat(0); 4];
for (i, (va, vb)) in a.iter_mut().zip(b.iter_mut()).enumerate() {
let wa = [
salt_words[0][i],
salt_words[1][i],
salt_words[2][i],
salt_words[3][i],
];
let wb = [
salt_words[4][i],
salt_words[5][i],
salt_words[6][i],
salt_words[7][i],
];
unsafe {
*va = v128_load(wa.as_ptr().cast::<v128>());
*vb = v128_load(wb.as_ptr().cast::<v128>());
}
}
(a, b)
}
#[target_feature(enable = "simd128")]
#[inline]
unsafe fn bcrypt_lanes_impl(
cost: u32,
key_words: &[[u32; 18]],
salt_words: &[[u32; 4]],
outs: &mut [[u8; 24]],
) {
debug_assert!((4..=31).contains(&cost));
debug_assert_eq!(key_words.len(), LANES);
debug_assert_eq!(salt_words.len(), LANES);
debug_assert_eq!(outs.len(), LANES);
#[cfg_attr(not(feature = "zeroize"), allow(unused_mut))]
let (mut kwa, mut kwb) = transpose_keys(key_words);
let (swa, swb) = transpose_salts(salt_words);
let mut state = State8x {
pa: [u32x4_splat(0); 18],
pb: [u32x4_splat(0); 18],
s: SBoxes8x([[0; 8]; 1024]),
};
expand_state_v8(&mut state, &swa, &swb, &kwa, &kwb);
for _ in 0..(1u64 << cost) {
expand0state_v8(&mut state, &kwa, &kwb);
expand0state_salt_v8(&mut state, &swa, &swb);
}
let mut ca = [
u32x4_splat(0x4f72_7068),
u32x4_splat(0x6561_6e42),
u32x4_splat(0x6568_6f6c),
u32x4_splat(0x6465_7253),
u32x4_splat(0x6372_7944),
u32x4_splat(0x6f75_6274),
];
let mut cb = ca;
for _ in 0..64 {
for pair in 0..3 {
let (la, ra, lb, rb) = encipher8(
&state,
ca[2 * pair],
ca[2 * pair + 1],
cb[2 * pair],
cb[2 * pair + 1],
);
ca[2 * pair] = la;
ca[2 * pair + 1] = ra;
cb[2 * pair] = lb;
cb[2 * pair + 1] = rb;
}
}
for w in 0..6 {
let mut words = [0u32; 8];
unsafe {
v128_store(words.as_mut_ptr().cast::<v128>(), ca[w]);
v128_store(words.as_mut_ptr().add(4).cast::<v128>(), cb[w]);
}
for (lane, out) in outs.iter_mut().enumerate() {
out[4 * w..4 * w + 4].copy_from_slice(&words[lane].to_be_bytes());
}
}
#[cfg(feature = "zeroize")]
{
crate::wipe::secure_wipe_u32(unsafe {
core::slice::from_raw_parts_mut(state.pa.as_mut_ptr().cast::<u32>(), 18 * 4)
});
crate::wipe::secure_wipe_u32(unsafe {
core::slice::from_raw_parts_mut(state.pb.as_mut_ptr().cast::<u32>(), 18 * 4)
});
crate::wipe::secure_wipe_u32(state.s.0.as_flattened_mut());
crate::wipe::secure_wipe_u32(unsafe {
core::slice::from_raw_parts_mut(ca.as_mut_ptr().cast::<u32>(), 6 * 4)
});
crate::wipe::secure_wipe_u32(unsafe {
core::slice::from_raw_parts_mut(cb.as_mut_ptr().cast::<u32>(), 6 * 4)
});
crate::wipe::secure_wipe_u32(unsafe {
core::slice::from_raw_parts_mut(kwa.as_mut_ptr().cast::<u32>(), 18 * 4)
});
crate::wipe::secure_wipe_u32(unsafe {
core::slice::from_raw_parts_mut(kwb.as_mut_ptr().cast::<u32>(), 18 * 4)
});
}
}
#[target_feature(enable = "simd128")]
pub unsafe fn bcrypt_lanes(
cost: u32,
key_words: &[[u32; 18]],
salt_words: &[[u32; 4]],
outs: &mut [[u8; 24]],
) {
debug_assert!((4..=31).contains(&cost));
debug_assert_eq!(key_words.len(), LANES);
debug_assert_eq!(salt_words.len(), LANES);
debug_assert_eq!(outs.len(), LANES);
unsafe { bcrypt_lanes_impl(cost, key_words, salt_words, outs) }
}
#[cfg(test)]
mod tests {
use super::*;
use crate::base64;
use crate::eks::{Backend, expand_key_words, salt_words};
fn key_words_for(password: &[u8]) -> [u32; 18] {
let mut key = [0u8; 72];
let copied = password.len().min(72);
key[..copied].copy_from_slice(&password[..copied]);
expand_key_words(&key, (copied + 1).min(72))
}
fn wasm128_available() -> bool {
!cfg!(miri) && Backend::Wasm128.is_available()
}
#[test]
fn dispatch_resolves_the_real_kernel() {
if !wasm128_available() {
return;
}
let wasm = crate::eks::bcrypt_lanes_fn(Backend::Wasm128);
let scalar = crate::eks::bcrypt_lanes_fn(Backend::Scalar);
assert!(
!core::ptr::fn_addr_eq(wasm, scalar),
"Backend::Wasm128 still resolves to the scalar kernel"
);
}
#[test]
fn lanes_match_scalar() {
if !wasm128_available() {
return;
}
let vector_salt =
base64::decode_16(b"CCCCCCCCCCCCCCCCCCCCC.").expect("test salt must decode");
let passwords: [&[u8]; 8] = [
b"",
b"U*U",
b"hunter2",
&[0xAA; 72],
b"correct horse battery staple",
&[0x00; 1],
b"\xff\xfe\xfd binary",
b"a",
];
let kws: [[u32; 18]; 8] = core::array::from_fn(|i| key_words_for(passwords[i]));
let sws: [[u32; 4]; 8] = [
salt_words(&[0x11; 16]),
salt_words(&vector_salt),
salt_words(&[0x33; 16]),
salt_words(&[0x44; 16]),
salt_words(&[0x55; 16]),
salt_words(&[0x66; 16]),
salt_words(&[0x77; 16]),
salt_words(&[0x88; 16]),
];
let mut outs = [[0u8; 24]; 8];
unsafe { bcrypt_lanes(5, &kws, &sws, &mut outs) };
for lane in 0..LANES {
let mut expected = [0u8; 24];
crate::eks::scalar::bcrypt_lanes(
5,
&kws[lane..lane + 1],
&sws[lane..lane + 1],
core::slice::from_mut(&mut expected),
);
assert_eq!(outs[lane], expected, "lane {lane} diverged from scalar");
}
}
}