use core::sync::atomic::{AtomicU8, Ordering};
use core::arch::x86_64::{
__m128i, _mm_add_epi32, _mm_cvtsi32_si128, _mm_cvtsi128_si32, _mm_loadu_si128, _mm_rol_epi32,
_mm_srli_epi64, _mm_srli_si128, _mm_ternarylogic_epi32, _mm_unpackhi_epi64,
};
use crate::consts::STATE_INIT;
const BLOCK_SIZE: usize = 64;
const STATE_WORDS: usize = 4;
const MD5_CONSTANTS: [u32; 64] = [
0xd76a_a478,
0xe8c7_b756,
0x2420_70db,
0xc1bd_ceee,
0xf57c_0faf,
0x4787_c62a,
0xa830_4613,
0xfd46_9501,
0x6980_98d8,
0x8b44_f7af,
0xffff_5bb1,
0x895c_d7be,
0x6b90_1122,
0xfd98_7193,
0xa679_438e,
0x49b4_0821,
0x124c_2332,
0x0d56_6e0c,
0xd8cf_331d,
0x3317_3e99,
0xf257_ec19,
0x8ea7_4a33,
0x1810_6d2d,
0x6a28_6dd8,
0xdbd9_7c15,
0x969c_d637,
0x0244_b8a2,
0x9d01_8293,
0x219a_3b68,
0xac4b_7772,
0x1cbd_c448,
0x8eed_de60,
0x00ea_6050,
0xaea0_c4e2,
0xc7bc_b26d,
0xe01a_22fe,
0x640a_d3e1,
0x29cb_28e5,
0x4447_69c5,
0x8f4c_4887,
0x4217_e194,
0xb7f3_0253,
0xbc7b_a81d,
0x473f_06d1,
0x59b1_4d5b,
0x7eb7_95c1,
0x3aae_3036,
0x4700_9677,
0x0987_fa4a,
0xe0c5_738d,
0x662b_7c56,
0xba1d_9c0d,
0xab74_aed9,
0xfc99_66f7,
0x9e79_260f,
0x4c6f_b437,
0xe836_87ce,
0x11b2_0358,
0x4130_380d,
0x4f9d_9113,
0x7e7f_bfde,
0x256c_92db,
0xadae_eb9b,
0xde8a_69e8,
];
#[inline]
pub(crate) fn is_supported() -> bool {
#[cfg(feature = "std")]
{
std::arch::is_x86_feature_detected!("avx512f")
&& std::arch::is_x86_feature_detected!("avx512vl")
}
#[cfg(not(feature = "std"))]
{
cfg!(all(target_feature = "avx512f", target_feature = "avx512vl"))
}
}
static PREFERRED_CACHE: AtomicU8 = AtomicU8::new(0);
#[inline]
pub(crate) fn is_preferred() -> bool {
match PREFERRED_CACHE.load(Ordering::Relaxed) {
1 => false,
2 => true,
_ => {
let preferred = is_supported() && is_genuine_intel();
PREFERRED_CACHE.store(if preferred { 2 } else { 1 }, Ordering::Relaxed);
preferred
}
}
}
#[inline]
#[allow(unused_unsafe)]
fn is_genuine_intel() -> bool {
let vendor = unsafe { core::arch::x86_64::__cpuid(0) };
vendor.ebx == 0x756e_6547 && vendor.edx == 0x4965_6e69 && vendor.ecx == 0x6c65_746e
}
#[inline(always)]
fn load_message_group(block: &[u8; BLOCK_SIZE], word: usize) -> __m128i {
debug_assert!(matches!(word, 0 | 4 | 8 | 12));
unsafe { _mm_loadu_si128(block.as_ptr().add(word * 4).cast::<__m128i>()) }
}
#[inline(always)]
fn load_constant_group(index: usize) -> __m128i {
debug_assert!(index <= 60 && index.is_multiple_of(4));
unsafe { _mm_loadu_si128(MD5_CONSTANTS.as_ptr().add(index).cast::<__m128i>()) }
}
macro_rules! round_x {
($truth:expr, $ia:expr, $a:ident, $b:expr, $c:expr, $input:expr, $shift:expr, $tmp2:ident) => {{
$a = _mm_add_epi32($ia, $input);
$tmp2 = _mm_ternarylogic_epi32::<$truth>($tmp2, $c, $b);
$a = _mm_add_epi32($a, $tmp2);
$a = _mm_rol_epi32::<$shift>($a);
$tmp2 = $c;
$a = _mm_add_epi32($a, $b);
}};
}
macro_rules! round_h {
($a:ident, $b:expr, $input:expr, $shift:expr, $tmp1:ident, $tmp2:ident) => {{
$tmp1 = _mm_add_epi32($a, $input);
$tmp2 = _mm_ternarylogic_epi32::<0x96>($tmp2, $a, $b);
$a = _mm_add_epi32($tmp1, $tmp2);
$a = _mm_rol_epi32::<$shift>($a);
$a = _mm_add_epi32($a, $b);
}};
}
macro_rules! rf4_first {
($v:expr, $a:ident, $b:ident, $c:ident, $d:ident, $ia:expr, $ib:expr, $ic:expr, $id:expr, $tmp1:ident, $tmp2:ident) => {{
let v = $v;
round_x!(0xd8, $ia, $a, $ib, $ic, v, 7, $tmp2);
$tmp1 = _mm_srli_epi64::<32>(v);
round_x!(0xd8, $id, $d, $a, $ib, $tmp1, 12, $tmp2);
$tmp1 = _mm_unpackhi_epi64(v, v);
round_x!(0xd8, $ic, $c, $d, $a, $tmp1, 17, $tmp2);
$tmp1 = _mm_srli_epi64::<32>($tmp1);
round_x!(0xd8, $ib, $b, $c, $d, $tmp1, 22, $tmp2);
}};
}
macro_rules! rf4 {
($v:expr, $a:ident, $b:ident, $c:ident, $d:ident, $tmp1:ident, $tmp2:ident) => {{
let v = $v;
round_x!(0xd8, $a, $a, $b, $c, v, 7, $tmp2);
$tmp1 = _mm_srli_epi64::<32>(v);
round_x!(0xd8, $d, $d, $a, $b, $tmp1, 12, $tmp2);
$tmp1 = _mm_unpackhi_epi64(v, v);
round_x!(0xd8, $c, $c, $d, $a, $tmp1, 17, $tmp2);
$tmp1 = _mm_srli_epi64::<32>($tmp1);
round_x!(0xd8, $b, $b, $c, $d, $tmp1, 22, $tmp2);
}};
}
macro_rules! rg4 {
($rs:expr, $r1:expr, $r2:expr, $a:ident, $b:ident, $c:ident, $d:ident, $tmp1:ident, $tmp2:ident) => {{
let rs = $rs;
let r1 = $r1;
let r2 = $r2;
$tmp1 = _mm_srli_epi64::<32>(rs);
round_x!(0xac, $a, $a, $b, $c, $tmp1, 5, $tmp2);
$tmp1 = _mm_unpackhi_epi64(r1, r1);
round_x!(0xac, $d, $d, $a, $b, $tmp1, 9, $tmp2);
$tmp1 = _mm_srli_si128::<12>(r2);
round_x!(0xac, $c, $c, $d, $a, $tmp1, 14, $tmp2);
round_x!(0xac, $b, $b, $c, $d, rs, 20, $tmp2);
}};
}
macro_rules! rh4 {
($r1:expr, $rs:expr, $r2:expr, $a:ident, $b:ident, $c:ident, $d:ident, $tmp1:ident, $tmp2:ident) => {{
let r1 = $r1;
let rs = $rs;
let r2 = $r2;
$tmp1 = _mm_srli_epi64::<32>(r1);
round_h!($a, $b, $tmp1, 4, $tmp1, $tmp2);
round_h!($d, $a, rs, 11, $tmp1, $tmp2);
$tmp1 = _mm_srli_si128::<12>(rs);
round_h!($c, $d, $tmp1, 16, $tmp1, $tmp2);
$tmp1 = _mm_unpackhi_epi64(r2, r2);
round_h!($b, $c, $tmp1, 23, $tmp1, $tmp2);
}};
}
macro_rules! ri4 {
($r1:expr, $rs:expr, $r2:expr, $a:ident, $b:ident, $c:ident, $d:ident, $tmp1:ident, $tmp2:ident) => {{
let r1 = $r1;
let rs = $rs;
let r2 = $r2;
round_x!(0x63, $a, $a, $b, $c, r1, 6, $tmp2);
$tmp1 = _mm_srli_si128::<12>(rs);
round_x!(0x63, $d, $d, $a, $b, $tmp1, 10, $tmp2);
$tmp1 = _mm_unpackhi_epi64(r2, r2);
round_x!(0x63, $c, $c, $d, $a, $tmp1, 15, $tmp2);
$tmp1 = _mm_srli_epi64::<32>(rs);
round_x!(0x63, $b, $b, $c, $d, $tmp1, 21, $tmp2);
}};
}
#[inline(always)]
#[allow(clippy::many_single_char_names)]
#[allow(clippy::too_many_lines)]
#[allow(unused_assignments)]
unsafe fn compress_block_vec(state: &mut [__m128i; STATE_WORDS], block: &[u8; BLOCK_SIZE]) {
unsafe {
let ia = state[0];
let ib = state[1];
let ic = state[2];
let id = state[3];
let mut a: __m128i;
let mut b: __m128i;
let mut c: __m128i;
let mut d: __m128i;
let mut tmp1;
let mut tmp2 = id;
let mut in0 = _mm_add_epi32(load_message_group(block, 0), load_constant_group(0));
let mut in4 = _mm_add_epi32(load_message_group(block, 4), load_constant_group(4));
let mut in8 = _mm_add_epi32(load_message_group(block, 8), load_constant_group(8));
let mut in12 = _mm_add_epi32(load_message_group(block, 12), load_constant_group(12));
rf4_first!(in0, a, b, c, d, ia, ib, ic, id, tmp1, tmp2);
rf4!(in4, a, b, c, d, tmp1, tmp2);
rf4!(in8, a, b, c, d, tmp1, tmp2);
rf4!(in12, a, b, c, d, tmp1, tmp2);
in0 = _mm_add_epi32(in0, load_constant_group(16));
in4 = _mm_add_epi32(in4, load_constant_group(20));
in8 = _mm_add_epi32(in8, load_constant_group(24));
rg4!(in0, in4, in8, a, b, c, d, tmp1, tmp2);
in12 = _mm_add_epi32(in12, load_constant_group(28));
rg4!(in4, in8, in12, a, b, c, d, tmp1, tmp2);
rg4!(in8, in12, in0, a, b, c, d, tmp1, tmp2);
rg4!(in12, in0, in4, a, b, c, d, tmp1, tmp2);
in4 = _mm_add_epi32(in4, load_constant_group(36));
tmp1 = _mm_srli_epi64::<32>(in4);
a = _mm_add_epi32(a, tmp1);
tmp2 = _mm_ternarylogic_epi32::<0x96>(tmp2, c, b);
a = _mm_add_epi32(a, tmp2);
a = _mm_rol_epi32::<4>(a);
a = _mm_add_epi32(a, b);
in8 = _mm_add_epi32(in8, load_constant_group(40));
round_h!(d, a, in8, 11, tmp1, tmp2);
tmp1 = _mm_srli_si128::<12>(in8);
in12 = _mm_add_epi32(in12, load_constant_group(44));
round_h!(c, d, tmp1, 16, tmp1, tmp2);
tmp1 = _mm_unpackhi_epi64(in12, in12);
round_h!(b, c, tmp1, 23, tmp1, tmp2);
in0 = _mm_add_epi32(in0, load_constant_group(32));
rh4!(in0, in4, in8, a, b, c, d, tmp1, tmp2);
rh4!(in12, in0, in4, a, b, c, d, tmp1, tmp2);
rh4!(in8, in12, in0, a, b, c, d, tmp1, tmp2);
tmp2 = d;
in0 = _mm_add_epi32(in0, load_constant_group(48));
in4 = _mm_add_epi32(in4, load_constant_group(52));
in12 = _mm_add_epi32(in12, load_constant_group(60));
ri4!(in0, in4, in12, a, b, c, d, tmp1, tmp2);
in8 = _mm_add_epi32(in8, load_constant_group(56));
ri4!(in12, in0, in8, a, b, c, d, tmp1, tmp2);
ri4!(in8, in12, in4, a, b, c, d, tmp1, tmp2);
ri4!(in4, in8, in0, a, b, c, d, tmp1, tmp2);
state[0] = _mm_add_epi32(a, ia);
state[1] = _mm_add_epi32(b, ib);
state[2] = _mm_add_epi32(c, ic);
state[3] = _mm_add_epi32(d, id);
}
}
#[inline(always)]
unsafe fn vector_state_from_scalar(state: &[u32; STATE_WORDS]) -> [__m128i; STATE_WORDS] {
unsafe {
[
_mm_cvtsi32_si128(state[0] as i32),
_mm_cvtsi32_si128(state[1] as i32),
_mm_cvtsi32_si128(state[2] as i32),
_mm_cvtsi32_si128(state[3] as i32),
]
}
}
#[inline(always)]
unsafe fn vector_state_to_scalar(state: [__m128i; STATE_WORDS]) -> [u32; STATE_WORDS] {
unsafe {
[
_mm_cvtsi128_si32(state[0]) as u32,
_mm_cvtsi128_si32(state[1]) as u32,
_mm_cvtsi128_si32(state[2]) as u32,
_mm_cvtsi128_si32(state[3]) as u32,
]
}
}
#[target_feature(enable = "avx512f,avx512vl")]
pub(crate) unsafe fn compress_block(state: &mut [u32; STATE_WORDS], block: &[u8; BLOCK_SIZE]) {
let mut vector_state = unsafe { vector_state_from_scalar(state) };
unsafe { compress_block_vec(&mut vector_state, block) };
*state = unsafe { vector_state_to_scalar(vector_state) };
}
#[cfg(feature = "digest")]
#[target_feature(enable = "avx512f,avx512vl")]
pub(crate) unsafe fn compress_blocks(state: &mut [u32; STATE_WORDS], blocks: &[[u8; BLOCK_SIZE]]) {
let mut vector_state = unsafe { vector_state_from_scalar(state) };
for block in blocks {
unsafe { compress_block_vec(&mut vector_state, block) };
}
*state = unsafe { vector_state_to_scalar(vector_state) };
}
#[target_feature(enable = "avx512f,avx512vl")]
pub(crate) unsafe fn hash(input: &[u8]) -> [u8; 16] {
let mut vector_state = unsafe { vector_state_from_scalar(&STATE_INIT) };
let mut chunks = input.chunks_exact(BLOCK_SIZE);
for chunk in &mut chunks {
let block: &[u8; BLOCK_SIZE] = chunk.try_into().expect("64-byte chunk");
unsafe { compress_block_vec(&mut vector_state, block) };
}
let tail = chunks.remainder();
let mut final_blocks = [[0u8; BLOCK_SIZE]; 2];
final_blocks[0][..tail.len()].copy_from_slice(tail);
final_blocks[0][tail.len()] = 0x80;
let bit_len = (input.len() as u64).wrapping_mul(8).to_le_bytes();
let used = if tail.len() <= 55 {
final_blocks[0][56..64].copy_from_slice(&bit_len);
1
} else {
final_blocks[1][56..64].copy_from_slice(&bit_len);
2
};
for block in &final_blocks[..used] {
unsafe { compress_block_vec(&mut vector_state, block) };
}
let state = unsafe { vector_state_to_scalar(vector_state) };
let mut out = [0u8; 16];
for (chunk, word) in out.chunks_exact_mut(4).zip(state) {
chunk.copy_from_slice(&word.to_le_bytes());
}
out
}