use crate::consts::{P_INIT, S_INIT};
const LANES: usize = 8;
const _: () = assert!(
LANES == super::Backend::Neon.lanes(),
"neon LANES and Backend::Neon.lanes() disagree"
);
const STRIDE: usize = 8;
#[repr(C, align(64))]
struct SBoxes([[u32; STRIDE]; 1024]);
struct State {
p: [[u32; STRIDE]; 18],
s: SBoxes,
}
#[inline(always)]
fn opaque(v: u32) -> u32 {
let mut v = v;
unsafe { core::arch::asm!("/* {0:w} */", inout(reg) v) };
v
}
#[inline(always)]
fn gather_idx(byte: u32, lane: u32) -> usize {
let mut i = (byte * STRIDE as u32) | lane;
unsafe {
core::arch::asm!("/* {0:w} */", inout(reg) i, options(preserves_flags, nostack, nomem))
};
i as usize
}
#[inline]
unsafe fn f_word(s: &SBoxes, w: u32, lane: u32) -> u32 {
let (a, b, c, d) = (w >> 24, (w >> 16) & 0xff, (w >> 8) & 0xff, w & 0xff);
unsafe {
let base = s.0.as_ptr().cast::<u32>();
let va = *base.add(gather_idx(a, lane));
let vb = *base.add(256 * STRIDE).add(gather_idx(b, lane));
let vc = *base.add(2 * 256 * STRIDE).add(gather_idx(c, lane));
let vd = *base.add(3 * 256 * STRIDE).add(gather_idx(d, lane));
(va.wrapping_add(vb) ^ vc).wrapping_add(vd)
}
}
macro_rules! rounds16_scalar {
($s:expr, $pw:expr, $((($la:ident, $ra:ident), ($lb:ident, $rb:ident), $lp:literal)),+ $(,)?) => {{
macro_rules! half2 {
($xa:ident, $ya:ident, $xb:ident, $yb:ident, $i:expr, $ln:literal) => {{
let p2 = unsafe { ($pw[$i].as_ptr().cast::<u64>()).add($ln).read_unaligned() };
let p2 = u64::from_le(p2);
$xa ^= p2 as u32;
$xb ^= (p2 >> 32) as u32;
$ya ^= opaque(unsafe { f_word($s, $xa, 2 * $ln) });
$yb ^= opaque(unsafe { f_word($s, $xb, 2 * $ln + 1) });
}};
}
for pair in 0..8 {
$(half2!($la, $ra, $lb, $rb, 2 * pair, $lp);)+
$(half2!($ra, $la, $rb, $lb, 2 * pair + 1, $lp);)+
}
$(
$la ^= $pw[16][2 * $lp];
$lb ^= $pw[16][2 * $lp + 1];
$ra ^= $pw[17][2 * $lp];
$rb ^= $pw[17][2 * $lp + 1];
core::mem::swap(&mut $la, &mut $ra);
core::mem::swap(&mut $lb, &mut $rb);
)+
}};
}
fn expand_state(state: &mut State, sw: &[[u32; STRIDE]; 4], kw: &[[u32; 18]; LANES]) {
for (i, p) in state.p.iter_mut().enumerate() {
for (w, k) in p[..LANES].iter_mut().zip(kw.iter()) {
*w = P_INIT[i] ^ k[i];
}
}
for (entry, &init) in state.s.0.iter_mut().zip(S_INIT.as_flattened().iter()) {
entry[..LANES].fill(init);
}
let (mut l0, mut l1, mut l2, mut l3, mut l4, mut l5, mut l6, mut l7) = (0, 0, 0, 0, 0, 0, 0, 0);
let (mut r0, mut r1, mut r2, mut r3, mut r4, mut r5, mut r6, mut r7) = (0, 0, 0, 0, 0, 0, 0, 0);
let mut j = 0usize;
for pair in 0..9 {
l0 ^= sw[j % 4][0];
l1 ^= sw[j % 4][1];
l2 ^= sw[j % 4][2];
l3 ^= sw[j % 4][3];
l4 ^= sw[j % 4][4];
l5 ^= sw[j % 4][5];
l6 ^= sw[j % 4][6];
l7 ^= sw[j % 4][7];
j += 1;
r0 ^= sw[j % 4][0];
r1 ^= sw[j % 4][1];
r2 ^= sw[j % 4][2];
r3 ^= sw[j % 4][3];
r4 ^= sw[j % 4][4];
r5 ^= sw[j % 4][5];
r6 ^= sw[j % 4][6];
r7 ^= sw[j % 4][7];
j += 1;
rounds16_scalar!(
&state.s, &state.p,
((l0, r0), (l1, r1), 0), ((l2, r2), (l3, r3), 1),
((l4, r4), (l5, r5), 2), ((l6, r6), (l7, r7), 3),
);
state.p[2 * pair][..LANES].copy_from_slice(&[l0, l1, l2, l3, l4, l5, l6, l7]);
state.p[2 * pair + 1][..LANES].copy_from_slice(&[r0, r1, r2, r3, r4, r5, r6, r7]);
}
for b in 0..4 {
for pair in 0..128 {
l0 ^= sw[j % 4][0];
l1 ^= sw[j % 4][1];
l2 ^= sw[j % 4][2];
l3 ^= sw[j % 4][3];
l4 ^= sw[j % 4][4];
l5 ^= sw[j % 4][5];
l6 ^= sw[j % 4][6];
l7 ^= sw[j % 4][7];
j += 1;
r0 ^= sw[j % 4][0];
r1 ^= sw[j % 4][1];
r2 ^= sw[j % 4][2];
r3 ^= sw[j % 4][3];
r4 ^= sw[j % 4][4];
r5 ^= sw[j % 4][5];
r6 ^= sw[j % 4][6];
r7 ^= sw[j % 4][7];
j += 1;
rounds16_scalar!(
&state.s, &state.p,
((l0, r0), (l1, r1), 0), ((l2, r2), (l3, r3), 1),
((l4, r4), (l5, r5), 2), ((l6, r6), (l7, r7), 3),
);
let e = b * 256 + 2 * pair;
state.s.0[e][..LANES].copy_from_slice(&[l0, l1, l2, l3, l4, l5, l6, l7]);
state.s.0[e + 1][..LANES].copy_from_slice(&[r0, r1, r2, r3, r4, r5, r6, r7]);
}
}
}
fn encrypt_zero_chain(state: &mut State) {
let (mut l0, mut l1, mut l2, mut l3, mut l4, mut l5, mut l6, mut l7) = (0, 0, 0, 0, 0, 0, 0, 0);
let (mut r0, mut r1, mut r2, mut r3, mut r4, mut r5, mut r6, mut r7) = (0, 0, 0, 0, 0, 0, 0, 0);
for pair in 0..9 {
rounds16_scalar!(
&state.s, &state.p,
((l0, r0), (l1, r1), 0), ((l2, r2), (l3, r3), 1),
((l4, r4), (l5, r5), 2), ((l6, r6), (l7, r7), 3),
);
state.p[2 * pair][..LANES].copy_from_slice(&[l0, l1, l2, l3, l4, l5, l6, l7]);
state.p[2 * pair + 1][..LANES].copy_from_slice(&[r0, r1, r2, r3, r4, r5, r6, r7]);
}
for b in 0..4 {
for pair in 0..128 {
rounds16_scalar!(
&state.s, &state.p,
((l0, r0), (l1, r1), 0), ((l2, r2), (l3, r3), 1),
((l4, r4), (l5, r5), 2), ((l6, r6), (l7, r7), 3),
);
let e = b * 256 + 2 * pair;
state.s.0[e][..LANES].copy_from_slice(&[l0, l1, l2, l3, l4, l5, l6, l7]);
state.s.0[e + 1][..LANES].copy_from_slice(&[r0, r1, r2, r3, r4, r5, r6, r7]);
}
}
}
fn expand0state(state: &mut State, kw: &[[u32; 18]; LANES]) {
for (i, p) in state.p.iter_mut().enumerate() {
for (w, k) in p[..LANES].iter_mut().zip(kw.iter()) {
*w ^= k[i];
}
}
encrypt_zero_chain(state);
}
fn expand0state_salt(state: &mut State, swk: &[[u32; 4]; LANES]) {
for (i, p) in state.p.iter_mut().enumerate() {
for (w, s) in p[..LANES].iter_mut().zip(swk.iter()) {
*w ^= s[i & 3];
}
}
encrypt_zero_chain(state);
}
#[inline]
unsafe fn bcrypt_lanes_impl(
cost: u32,
key_words: &[[u32; 18]],
salt_words: &[[u32; 4]],
outs: &mut [[u8; 24]],
) {
debug_assert!((4..=31).contains(&cost));
debug_assert_eq!(key_words.len(), LANES);
debug_assert_eq!(salt_words.len(), LANES);
debug_assert_eq!(outs.len(), LANES);
#[cfg_attr(not(feature = "zeroize"), allow(unused_mut))]
let mut kw: [[u32; 18]; LANES] = core::array::from_fn(|l| key_words[l]);
let swk: [[u32; 4]; LANES] = core::array::from_fn(|l| salt_words[l]);
let mut sw = [[0u32; STRIDE]; 4];
for (w, row) in sw.iter_mut().enumerate() {
for (slot, s) in row[..LANES].iter_mut().zip(salt_words.iter()) {
*slot = s[w];
}
}
let mut state = State {
p: [[0; STRIDE]; 18],
s: SBoxes([[0; STRIDE]; 1024]),
};
expand_state(&mut state, &sw, &kw);
for _ in 0..(1u64 << cost) {
expand0state(&mut state, &kw);
expand0state_salt(&mut state, &swk);
}
let mut cdata = [[0u32; STRIDE]; 6];
for (row, init) in cdata.iter_mut().zip([
0x4f72_7068,
0x6561_6e42,
0x6568_6f6c,
0x6465_7253,
0x6372_7944,
0x6f75_6274,
]) {
row[..LANES].fill(init);
}
for _ in 0..64 {
for pair in 0..3 {
let (mut l0, mut l1, mut l2, mut l3, mut l4, mut l5, mut l6, mut l7) = (
cdata[2 * pair][0],
cdata[2 * pair][1],
cdata[2 * pair][2],
cdata[2 * pair][3],
cdata[2 * pair][4],
cdata[2 * pair][5],
cdata[2 * pair][6],
cdata[2 * pair][7],
);
let (mut r0, mut r1, mut r2, mut r3, mut r4, mut r5, mut r6, mut r7) = (
cdata[2 * pair + 1][0],
cdata[2 * pair + 1][1],
cdata[2 * pair + 1][2],
cdata[2 * pair + 1][3],
cdata[2 * pair + 1][4],
cdata[2 * pair + 1][5],
cdata[2 * pair + 1][6],
cdata[2 * pair + 1][7],
);
rounds16_scalar!(
&state.s, &state.p,
((l0, r0), (l1, r1), 0), ((l2, r2), (l3, r3), 1),
((l4, r4), (l5, r5), 2), ((l6, r6), (l7, r7), 3),
);
cdata[2 * pair][..LANES].copy_from_slice(&[l0, l1, l2, l3, l4, l5, l6, l7]);
cdata[2 * pair + 1][..LANES].copy_from_slice(&[r0, r1, r2, r3, r4, r5, r6, r7]);
}
}
for (w, row) in cdata.iter().enumerate() {
for (lane, out) in outs.iter_mut().enumerate() {
out[4 * w..4 * w + 4].copy_from_slice(&row[lane].to_be_bytes());
}
}
#[cfg(feature = "zeroize")]
{
crate::wipe::secure_wipe_u32(state.p.as_flattened_mut());
crate::wipe::secure_wipe_u32(state.s.0.as_flattened_mut());
crate::wipe::secure_wipe_u32(cdata.as_flattened_mut());
crate::wipe::secure_wipe_u32(kw.as_flattened_mut());
}
}
#[inline]
pub unsafe fn bcrypt_lanes(
cost: u32,
key_words: &[[u32; 18]],
salt_words: &[[u32; 4]],
outs: &mut [[u8; 24]],
) {
debug_assert!((4..=31).contains(&cost));
debug_assert_eq!(key_words.len(), LANES);
debug_assert_eq!(salt_words.len(), LANES);
debug_assert_eq!(outs.len(), LANES);
unsafe { bcrypt_lanes_impl(cost, key_words, salt_words, outs) }
}
#[cfg(test)]
mod tests {
use super::*;
use crate::base64;
use crate::eks::{expand_key_words, salt_words};
fn key_words_for(password: &[u8]) -> [u32; 18] {
let mut key = [0u8; 72];
let copied = password.len().min(72);
key[..copied].copy_from_slice(&password[..copied]);
expand_key_words(&key, (copied + 1).min(72))
}
#[cfg(not(miri))]
#[test]
fn lanes_match_scalar() {
let vector_salt =
base64::decode_16(b"CCCCCCCCCCCCCCCCCCCCC.").expect("test salt must decode");
let passwords: [&[u8]; LANES] = [
b"",
b"U*U",
b"hunter2",
&[0xAA; 72],
b"bcrypt",
&[0x55; 71],
b"OrpheanBeholderScryDoubt",
&[0x01],
];
let kws: [[u32; 18]; LANES] = core::array::from_fn(|i| key_words_for(passwords[i]));
let sws: [[u32; 4]; LANES] = [
salt_words(&[0x11; 16]),
salt_words(&vector_salt),
salt_words(&[0x33; 16]),
salt_words(&[0x44; 16]),
salt_words(&[0x55; 16]),
salt_words(&[0x66; 16]),
salt_words(&[0x77; 16]),
salt_words(&[0x88; 16]),
];
let mut outs = [[0u8; 24]; LANES];
unsafe { bcrypt_lanes(5, &kws, &sws, &mut outs) };
for lane in 0..LANES {
let mut expected = [0u8; 24];
crate::eks::scalar::bcrypt_lanes(
5,
&kws[lane..lane + 1],
&sws[lane..lane + 1],
core::slice::from_mut(&mut expected),
);
assert_eq!(outs[lane], expected, "lane {lane} diverged from scalar");
}
}
}