pub fn shuffle2x_buffer_size(slice_words: usize) -> usize {
assert!(
slice_words.is_multiple_of(16),
"shuffle2x_buffer_size: slice_words ({slice_words}) must be a multiple of 16"
);
slice_words * 2
}
pub fn to_shuffle2x(src: &[u8], dst: &mut [u8]) {
assert_eq!(
src.len(),
dst.len(),
"to_shuffle2x: src and dst must have equal length"
);
assert!(
src.len().is_multiple_of(32),
"to_shuffle2x: src length ({}) must be a multiple of 32",
src.len()
);
#[cfg(target_arch = "x86_64")]
if std::is_x86_feature_detected!("avx2") {
unsafe { to_shuffle2x_avx2(src, dst) };
return;
}
to_shuffle2x_scalar(src, dst);
}
pub fn from_shuffle2x(src: &[u8], dst: &mut [u8]) {
assert_eq!(
src.len(),
dst.len(),
"from_shuffle2x: src and dst must have equal length"
);
assert!(
src.len().is_multiple_of(32),
"from_shuffle2x: src length ({}) must be a multiple of 32",
src.len()
);
#[cfg(target_arch = "x86_64")]
if std::is_x86_feature_detected!("avx2") {
unsafe { from_shuffle2x_avx2(src, dst) };
return;
}
from_shuffle2x_scalar(src, dst);
}
fn to_shuffle2x_scalar(src: &[u8], dst: &mut [u8]) {
for (chunk_in, chunk_out) in src
.as_chunks::<32>()
.0
.iter()
.zip(dst.as_chunks_mut::<32>().0)
{
for i in 0..16 {
chunk_out[i] = chunk_in[i * 2]; chunk_out[16 + i] = chunk_in[i * 2 + 1]; }
}
}
fn from_shuffle2x_scalar(src: &[u8], dst: &mut [u8]) {
for (chunk_in, chunk_out) in src
.as_chunks::<32>()
.0
.iter()
.zip(dst.as_chunks_mut::<32>().0)
{
for i in 0..16 {
chunk_out[i * 2] = chunk_in[i]; chunk_out[i * 2 + 1] = chunk_in[16 + i]; }
}
}
#[cfg(target_arch = "x86_64")]
#[target_feature(enable = "avx2")]
unsafe fn to_shuffle2x_avx2(src: &[u8], dst: &mut [u8]) {
use std::arch::x86_64::*;
let sep_mask = _mm256_set_epi8(
15, 13, 11, 9, 7, 5, 3, 1, 14, 12, 10, 8, 6, 4, 2, 0, 15, 13, 11, 9, 7, 5, 3, 1, 14, 12, 10, 8, 6, 4, 2, 0, );
for (chunk_in, chunk_out) in src
.as_chunks::<32>()
.0
.iter()
.zip(dst.as_chunks_mut::<32>().0)
{
let v = _mm256_loadu_si256(chunk_in.as_ptr() as *const __m256i);
let separated = _mm256_shuffle_epi8(v, sep_mask);
let result = _mm256_permute4x64_epi64(separated, 0xD8);
_mm256_storeu_si256(chunk_out.as_mut_ptr() as *mut __m256i, result);
}
}
#[cfg(target_arch = "x86_64")]
#[target_feature(enable = "avx2")]
unsafe fn from_shuffle2x_avx2(src: &[u8], dst: &mut [u8]) {
use std::arch::x86_64::*;
let interleave_mask = _mm256_set_epi8(
15, 7, 14, 6, 13, 5, 12, 4, 11, 3, 10, 2, 9, 1, 8, 0, 15, 7, 14, 6, 13, 5, 12, 4, 11, 3, 10, 2, 9, 1, 8, 0, );
for (chunk_in, chunk_out) in src
.as_chunks::<32>()
.0
.iter()
.zip(dst.as_chunks_mut::<32>().0)
{
let v = _mm256_loadu_si256(chunk_in.as_ptr() as *const __m256i);
let unpermed = _mm256_permute4x64_epi64(v, 0xD8);
let result = _mm256_shuffle_epi8(unpermed, interleave_mask);
_mm256_storeu_si256(chunk_out.as_mut_ptr() as *mut __m256i, result);
}
}
#[cfg(test)]
mod tests {
use super::*;
fn roundtrip(bytes: &[u8]) {
assert!(bytes.len().is_multiple_of(32));
let mut s2x = vec![0u8; bytes.len()];
let mut recovered = vec![0u8; bytes.len()];
to_shuffle2x(bytes, &mut s2x);
from_shuffle2x(&s2x, &mut recovered);
assert_eq!(
recovered,
bytes,
"round-trip failed for {} bytes",
bytes.len()
);
}
#[test]
fn roundtrip_zeros() {
roundtrip(&[0u8; 32]);
roundtrip(&[0u8; 128]);
}
#[test]
fn roundtrip_all_ones() {
roundtrip(&[0xFFu8; 32]);
roundtrip(&[0xFFu8; 256]);
}
#[test]
fn roundtrip_incrementing() {
let bytes: Vec<u8> = (0..256u16).flat_map(|i| i.to_le_bytes()).collect();
roundtrip(&bytes);
}
#[test]
fn roundtrip_random() {
let mut lcg: u64 = 0xDEAD_BEEF_1234_5678;
let bytes: Vec<u8> = (0..512)
.map(|_| {
lcg = lcg
.wrapping_mul(6364136223846793005)
.wrapping_add(1442695040888963407);
(lcg >> 56) as u8
})
.collect();
roundtrip(&bytes);
}
#[test]
fn layout_correct_lo_hi_separation() {
let bytes: Vec<u8> = (0u8..32).collect();
let mut s2x = vec![0u8; 32];
to_shuffle2x(&bytes, &mut s2x);
for i in 0..16usize {
assert_eq!(
s2x[i],
(i * 2) as u8,
"lane0[{i}] should be lo byte of word {i}"
);
assert_eq!(
s2x[16 + i],
(i * 2 + 1) as u8,
"lane1[{i}] should be hi byte of word {i}"
);
}
}
#[test]
#[allow(unreachable_code)]
fn scalar_and_simd_agree() {
#[cfg(not(target_arch = "x86_64"))]
return;
let mut lcg: u64 = 0xCAFE_BABE_1234;
let bytes: Vec<u8> = (0..256)
.map(|_| {
lcg = lcg
.wrapping_mul(6364136223846793005)
.wrapping_add(1442695040888963407);
(lcg >> 56) as u8
})
.collect();
let mut out_scalar = vec![0u8; 256];
let mut out_simd = vec![0u8; 256];
to_shuffle2x_scalar(&bytes, &mut out_scalar);
to_shuffle2x(&bytes, &mut out_simd);
assert_eq!(out_scalar, out_simd, "to_shuffle2x: scalar vs SIMD differ");
let mut inv_scalar = vec![0u8; 256];
let mut inv_simd = vec![0u8; 256];
from_shuffle2x_scalar(&out_scalar, &mut inv_scalar);
from_shuffle2x(&out_simd, &mut inv_simd);
assert_eq!(
inv_scalar, inv_simd,
"from_shuffle2x: scalar vs SIMD differ"
);
}
}