use core::arch::x86_64::{
__m128i, __m256i, _mm_add_epi8, _mm_and_si128, _mm_andnot_si128, _mm_cmpeq_epi8,
_mm_cmpgt_epi8, _mm_loadu_si128, _mm_madd_epi16, _mm_maddubs_epi16, _mm_movemask_epi8,
_mm_mulhi_epu16, _mm_mullo_epi16, _mm_or_si128, _mm_set1_epi32, _mm_set1_epi8,
_mm_setr_epi8, _mm_setzero_si128, _mm_shuffle_epi8, _mm_slli_si128, _mm_srli_epi16,
_mm_srli_si128, _mm_storeu_si128, _mm256_add_epi8, _mm256_and_si256, _mm256_andnot_si256,
_mm256_castsi128_si256, _mm256_cmpeq_epi8, _mm256_cmpgt_epi8, _mm256_inserti128_si256,
_mm256_madd_epi16, _mm256_maddubs_epi16, _mm256_movemask_epi8, _mm256_mulhi_epu16,
_mm256_mullo_epi16, _mm256_or_si256, _mm256_permutevar8x32_epi32, _mm256_set1_epi32,
_mm256_set1_epi8, _mm256_setr_epi32, _mm256_setr_epi8, _mm256_setzero_si256,
_mm256_shuffle_epi8, _mm256_srli_epi16, _mm256_storeu_si256,
};
use super::{HASH_B64_LEN, SALT_B64_LEN, scalar};
#[cfg(feature = "std")]
#[inline]
fn have_avx2() -> bool {
!cfg!(miri) && std::arch::is_x86_feature_detected!("avx2")
}
#[cfg(not(feature = "std"))]
#[inline]
fn have_avx2() -> bool {
!cfg!(miri) && cfg!(target_feature = "avx2")
}
#[cfg(feature = "std")]
#[inline]
fn have_ssse3() -> bool {
!cfg!(miri) && std::arch::is_x86_feature_detected!("ssse3")
}
#[cfg(not(feature = "std"))]
#[inline]
fn have_ssse3() -> bool {
!cfg!(miri) && cfg!(target_feature = "ssse3")
}
#[target_feature(enable = "ssse3")]
#[inline]
fn enc_block_ssse3(x: __m128i) -> __m128i {
let shuffled = _mm_shuffle_epi8(
x,
_mm_setr_epi8(1, 0, 2, 1, 4, 3, 5, 4, 7, 6, 8, 7, 10, 9, 11, 10),
);
let t0 = _mm_and_si128(shuffled, _mm_set1_epi32(0x0FC0_FC00));
let t1 = _mm_mulhi_epu16(t0, _mm_set1_epi32(0x0400_0040));
let t2 = _mm_and_si128(shuffled, _mm_set1_epi32(0x003F_03F0));
let t3 = _mm_mullo_epi16(t2, _mm_set1_epi32(0x0100_0010));
let v = _mm_or_si128(t1, t3);
let m1 = _mm_cmpgt_epi8(v, _mm_set1_epi8(1));
let m2 = _mm_cmpgt_epi8(v, _mm_set1_epi8(27));
let m3 = _mm_cmpgt_epi8(v, _mm_set1_epi8(53));
let mut off = _mm_set1_epi8(46);
off = _mm_or_si128(_mm_and_si128(m1, _mm_set1_epi8(63)), _mm_andnot_si128(m1, off));
off = _mm_or_si128(_mm_and_si128(m2, _mm_set1_epi8(69)), _mm_andnot_si128(m2, off));
off = _mm_or_si128(_mm_and_si128(m3, _mm_set1_epi8(-6)), _mm_andnot_si128(m3, off));
_mm_add_epi8(v, off)
}
#[target_feature(enable = "avx2")]
#[inline]
fn enc_block_avx2(x: __m256i) -> __m256i {
let shuffled = _mm256_shuffle_epi8(
x,
_mm256_setr_epi8(
1, 0, 2, 1, 4, 3, 5, 4, 7, 6, 8, 7, 10, 9, 11, 10, 1, 0, 2, 1, 4, 3, 5, 4, 7, 6,
8, 7, 10, 9, 11, 10,
),
);
let t0 = _mm256_and_si256(shuffled, _mm256_set1_epi32(0x0FC0_FC00));
let t1 = _mm256_mulhi_epu16(t0, _mm256_set1_epi32(0x0400_0040));
let t2 = _mm256_and_si256(shuffled, _mm256_set1_epi32(0x003F_03F0));
let t3 = _mm256_mullo_epi16(t2, _mm256_set1_epi32(0x0100_0010));
let v = _mm256_or_si256(t1, t3);
let m1 = _mm256_cmpgt_epi8(v, _mm256_set1_epi8(1));
let m2 = _mm256_cmpgt_epi8(v, _mm256_set1_epi8(27));
let m3 = _mm256_cmpgt_epi8(v, _mm256_set1_epi8(53));
let mut off = _mm256_set1_epi8(46);
off = _mm256_or_si256(
_mm256_and_si256(m1, _mm256_set1_epi8(63)),
_mm256_andnot_si256(m1, off),
);
off = _mm256_or_si256(
_mm256_and_si256(m2, _mm256_set1_epi8(69)),
_mm256_andnot_si256(m2, off),
);
off = _mm256_or_si256(
_mm256_and_si256(m3, _mm256_set1_epi8(-6)),
_mm256_andnot_si256(m3, off),
);
_mm256_add_epi8(v, off)
}
#[target_feature(enable = "ssse3")]
unsafe fn encode_ssse3<const IN: usize, const OUT: usize>(bytes: &[u8; IN]) -> [u8; OUT] {
let mut buf = [0u8; 32];
buf[..IN].copy_from_slice(bytes);
let (lo, hi) = unsafe {
(
_mm_loadu_si128(buf.as_ptr().cast::<__m128i>()),
_mm_loadu_si128(buf.as_ptr().add(12).cast::<__m128i>()),
)
};
let chars_lo = enc_block_ssse3(lo);
let chars_hi = enc_block_ssse3(hi);
let mut tmp = [0u8; 32];
unsafe {
_mm_storeu_si128(tmp.as_mut_ptr().cast::<__m128i>(), chars_lo);
_mm_storeu_si128(tmp.as_mut_ptr().add(16).cast::<__m128i>(), chars_hi);
}
let mut out = [0u8; OUT];
out.copy_from_slice(&tmp[..OUT]);
out
}
#[target_feature(enable = "avx2")]
unsafe fn encode_avx2<const IN: usize, const OUT: usize>(bytes: &[u8; IN]) -> [u8; OUT] {
let mut buf = [0u8; 32];
buf[..IN].copy_from_slice(bytes);
let x = unsafe {
let lo = _mm_loadu_si128(buf.as_ptr().cast::<__m128i>());
let hi = _mm_loadu_si128(buf.as_ptr().add(12).cast::<__m128i>());
_mm256_inserti128_si256::<1>(_mm256_castsi128_si256(lo), hi)
};
let chars = enc_block_avx2(x);
let mut tmp = [0u8; 32];
unsafe { _mm256_storeu_si256(tmp.as_mut_ptr().cast::<__m256i>(), chars) };
let mut out = [0u8; OUT];
out.copy_from_slice(&tmp[..OUT]);
out
}
#[target_feature(enable = "ssse3")]
#[inline]
fn dec_translate_ssse3(c: __m128i) -> (__m128i, __m128i) {
let nibble = _mm_set1_epi8(0x0F);
let lo = _mm_and_si128(c, nibble);
let hi = _mm_and_si128(_mm_srli_epi16::<4>(c), nibble);
let lo_tab = _mm_setr_epi8(
0x2A, 0x3E, 0x3E, 0x3E, 0x3E, 0x3E, 0x3E, 0x3E, 0x3E, 0x3E, 0x3C, 0x14, 0x14, 0x14,
0x15, 0x15,
);
let hi_tab = _mm_setr_epi8(
0x00, 0x00, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00,
);
let off_tab = _mm_setr_epi8(0, 0, -46, 6, -63, -63, -69, -69, 0, 0, 0, 0, 0, 0, 0, 0);
let bits = _mm_and_si128(_mm_shuffle_epi8(lo_tab, lo), _mm_shuffle_epi8(hi_tab, hi));
let bad = _mm_cmpeq_epi8(bits, _mm_setzero_si128());
let v = _mm_add_epi8(c, _mm_shuffle_epi8(off_tab, hi));
(v, bad)
}
#[target_feature(enable = "avx2")]
#[inline]
fn dec_translate_avx2(c: __m256i) -> (__m256i, __m256i) {
let nibble = _mm256_set1_epi8(0x0F);
let lo = _mm256_and_si256(c, nibble);
let hi = _mm256_and_si256(_mm256_srli_epi16::<4>(c), nibble);
let lo_tab = _mm256_setr_epi8(
0x2A, 0x3E, 0x3E, 0x3E, 0x3E, 0x3E, 0x3E, 0x3E, 0x3E, 0x3E, 0x3C, 0x14, 0x14, 0x14,
0x15, 0x15, 0x2A, 0x3E, 0x3E, 0x3E, 0x3E, 0x3E, 0x3E, 0x3E, 0x3E, 0x3E, 0x3C, 0x14,
0x14, 0x14, 0x15, 0x15,
);
let hi_tab = _mm256_setr_epi8(
0x00, 0x00, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00,
);
let off_tab = _mm256_setr_epi8(
0, 0, -46, 6, -63, -63, -69, -69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -46, 6, -63, -63, -69,
-69, 0, 0, 0, 0, 0, 0, 0, 0,
);
let bits = _mm256_and_si256(
_mm256_shuffle_epi8(lo_tab, lo),
_mm256_shuffle_epi8(hi_tab, hi),
);
let bad = _mm256_cmpeq_epi8(bits, _mm256_setzero_si256());
let v = _mm256_add_epi8(c, _mm256_shuffle_epi8(off_tab, hi));
(v, bad)
}
#[target_feature(enable = "ssse3")]
#[inline]
fn dec_tail_ssse3<const CHARS: usize>(s: &[u8]) -> __m128i {
assert!(CHARS == SALT_B64_LEN || CHARS == HASH_B64_LEN);
let last16 = unsafe { _mm_loadu_si128(s.as_ptr().add(CHARS - 16).cast::<__m128i>()) };
let dots = _mm_set1_epi8(b'.' as i8);
if CHARS == SALT_B64_LEN {
_mm_or_si128(
_mm_srli_si128::<{ (32 - SALT_B64_LEN) as i32 }>(last16),
_mm_slli_si128::<{ (SALT_B64_LEN - 16) as i32 }>(dots),
)
} else {
_mm_or_si128(
_mm_srli_si128::<{ (32 - HASH_B64_LEN) as i32 }>(last16),
_mm_slli_si128::<{ (HASH_B64_LEN - 16) as i32 }>(dots),
)
}
}
#[target_feature(enable = "ssse3")]
#[inline]
fn dec_pack_ssse3(v: __m128i) -> __m128i {
let merge = _mm_maddubs_epi16(v, _mm_set1_epi32(0x0140_0140));
let n = _mm_madd_epi16(merge, _mm_set1_epi32(0x0001_1000));
_mm_shuffle_epi8(
n,
_mm_setr_epi8(2, 1, 0, 6, 5, 4, 10, 9, 8, 14, 13, 12, -1, -1, -1, -1),
)
}
#[target_feature(enable = "avx2")]
#[inline]
fn dec_pack_avx2(v: __m256i) -> __m256i {
let merge = _mm256_maddubs_epi16(v, _mm256_set1_epi32(0x0140_0140));
let n = _mm256_madd_epi16(merge, _mm256_set1_epi32(0x0001_1000));
let bytes = _mm256_shuffle_epi8(
n,
_mm256_setr_epi8(
2, 1, 0, 6, 5, 4, 10, 9, 8, 14, 13, 12, -1, -1, -1, -1, 2, 1, 0, 6, 5, 4, 10, 9,
8, 14, 13, 12, -1, -1, -1, -1,
),
);
_mm256_permutevar8x32_epi32(bytes, _mm256_setr_epi32(0, 1, 2, 4, 5, 6, -1, -1))
}
#[target_feature(enable = "ssse3")]
unsafe fn decode_ssse3<const CHARS: usize, const OUT: usize>(
s: &[u8],
) -> Result<[u8; OUT], ()> {
if s.len() != CHARS {
return Err(());
}
let lo = unsafe { _mm_loadu_si128(s.as_ptr().cast::<__m128i>()) };
let hi = dec_tail_ssse3::<CHARS>(s);
let (v0, bad0) = dec_translate_ssse3(lo);
let (v1, bad1) = dec_translate_ssse3(hi);
if _mm_movemask_epi8(_mm_or_si128(bad0, bad1)) != 0 {
return Err(());
}
let bytes_lo = dec_pack_ssse3(v0);
let bytes_hi = dec_pack_ssse3(v1);
let mut tmp = [0u8; 32];
unsafe {
_mm_storeu_si128(tmp.as_mut_ptr().cast::<__m128i>(), bytes_lo);
_mm_storeu_si128(tmp.as_mut_ptr().add(12).cast::<__m128i>(), bytes_hi);
}
let mut out = [0u8; OUT];
out.copy_from_slice(&tmp[..OUT]);
Ok(out)
}
#[target_feature(enable = "avx2")]
unsafe fn decode_avx2<const CHARS: usize, const OUT: usize>(
s: &[u8],
) -> Result<[u8; OUT], ()> {
if s.len() != CHARS {
return Err(());
}
let x = unsafe {
let lo = _mm_loadu_si128(s.as_ptr().cast::<__m128i>());
_mm256_inserti128_si256::<1>(_mm256_castsi128_si256(lo), dec_tail_ssse3::<CHARS>(s))
};
let (v, bad) = dec_translate_avx2(x);
if _mm256_movemask_epi8(bad) != 0 {
return Err(());
}
let packed = dec_pack_avx2(v);
let mut tmp = [0u8; 32];
unsafe { _mm256_storeu_si256(tmp.as_mut_ptr().cast::<__m256i>(), packed) };
let mut out = [0u8; OUT];
out.copy_from_slice(&tmp[..OUT]);
Ok(out)
}
pub(crate) fn encode_16(bytes: &[u8; 16]) -> [u8; SALT_B64_LEN] {
if have_ssse3() {
unsafe { encode_ssse3::<16, SALT_B64_LEN>(bytes) }
} else if have_avx2() {
unsafe { encode_avx2::<16, SALT_B64_LEN>(bytes) }
} else {
scalar::encode_16(bytes)
}
}
pub(crate) fn encode_23(bytes: &[u8; 23]) -> [u8; HASH_B64_LEN] {
if have_ssse3() {
unsafe { encode_ssse3::<23, HASH_B64_LEN>(bytes) }
} else if have_avx2() {
unsafe { encode_avx2::<23, HASH_B64_LEN>(bytes) }
} else {
scalar::encode_23(bytes)
}
}
pub(crate) fn decode_16(s: &[u8]) -> Result<[u8; 16], ()> {
if have_avx2() {
unsafe { decode_avx2::<SALT_B64_LEN, 16>(s) }
} else if have_ssse3() {
unsafe { decode_ssse3::<SALT_B64_LEN, 16>(s) }
} else {
scalar::decode_16(s)
}
}
pub(crate) fn decode_23(s: &[u8]) -> Result<[u8; 23], ()> {
if have_avx2() {
unsafe { decode_avx2::<HASH_B64_LEN, 23>(s) }
} else if have_ssse3() {
unsafe { decode_ssse3::<HASH_B64_LEN, 23>(s) }
} else {
scalar::decode_23(s)
}
}
#[cfg(test)]
mod tests {
use super::*;
struct Rng(u64);
impl Rng {
fn fill(&mut self, buf: &mut [u8]) {
for b in buf {
let mut x = self.0;
x ^= x >> 12;
x ^= x << 25;
x ^= x >> 27;
self.0 = x;
*b = (x.wrapping_mul(0x2545_F491_4F6C_DD1D) >> 56) as u8;
}
}
}
macro_rules! check_encode {
($kernel:ident) => {{
for pos in 0..16 {
for b in 0..=u8::MAX {
let mut input = [0u8; 16];
input[pos] = b;
assert_eq!($kernel::<16, SALT_B64_LEN>(&input), scalar::encode_16(&input));
}
}
for pos in 0..23 {
for b in 0..=u8::MAX {
let mut input = [0u8; 23];
input[pos] = b;
assert_eq!($kernel::<23, HASH_B64_LEN>(&input), scalar::encode_23(&input));
}
}
let mut rng = Rng(0xC0FF_EE12_3456_7890);
let mut b16 = [0u8; 16];
let mut b23 = [0u8; 23];
for _ in 0..512 {
rng.fill(&mut b16);
rng.fill(&mut b23);
assert_eq!($kernel::<16, SALT_B64_LEN>(&b16), scalar::encode_16(&b16));
assert_eq!($kernel::<23, HASH_B64_LEN>(&b23), scalar::encode_23(&b23));
}
}};
}
macro_rules! check_decode {
($kernel:ident) => {{
let base16 = *b"CCCCCCCCCCCCCCCCCCCCCC";
let base23 = *b"CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC";
for pos in 0..SALT_B64_LEN {
for c in 0..=u8::MAX {
let mut s = base16;
s[pos] = c;
assert_eq!(
$kernel::<SALT_B64_LEN, 16>(&s),
scalar::decode_16(&s),
"16: pos {pos} c {c:#04x}"
);
}
}
for pos in 0..HASH_B64_LEN {
for c in 0..=u8::MAX {
let mut s = base23;
s[pos] = c;
assert_eq!(
$kernel::<HASH_B64_LEN, 23>(&s),
scalar::decode_23(&s),
"23: pos {pos} c {c:#04x}"
);
}
}
let s = [b'C'; 33];
for len in 0..=33usize {
assert_eq!($kernel::<SALT_B64_LEN, 16>(&s[..len]), scalar::decode_16(&s[..len]));
assert_eq!($kernel::<HASH_B64_LEN, 23>(&s[..len]), scalar::decode_23(&s[..len]));
}
let mut rng = Rng(0xB529_7A4D_1D2B_9F83);
let mut b16 = [0u8; 16];
let mut b23 = [0u8; 23];
for _ in 0..256 {
rng.fill(&mut b16);
rng.fill(&mut b23);
assert_eq!($kernel::<SALT_B64_LEN, 16>(&scalar::encode_16(&b16)), Ok(b16));
assert_eq!($kernel::<HASH_B64_LEN, 23>(&scalar::encode_23(&b23)), Ok(b23));
}
}};
}
#[test]
fn avx2_matches_scalar() {
if !have_avx2() {
return;
}
unsafe {
check_encode!(encode_avx2);
check_decode!(decode_avx2);
}
}
#[test]
fn ssse3_matches_scalar() {
if !have_ssse3() {
return;
}
unsafe {
check_encode!(encode_ssse3);
check_decode!(decode_ssse3);
}
}
}