use std::arch::x86_64::{_mm_crc32_u8, _mm_crc32_u64};
use std::sync::OnceLock;
use super::combine::combine;
use super::scalar::split_chunk;
const THREE_WAY_THRESHOLD: usize = 32 * 1024;
#[inline]
pub(crate) fn is_available() -> bool {
static AVAILABLE: OnceLock<bool> = OnceLock::new();
*AVAILABLE.get_or_init(|| std::arch::is_x86_feature_detected!("sse4.2"))
}
#[target_feature(enable = "sse4.2")]
pub(crate) unsafe fn update_state_hw(state: u32, data: &[u8]) -> u32 {
if data.len() >= THREE_WAY_THRESHOLD {
interleaved(state, data)
} else {
single_stream(state, data)
}
}
#[target_feature(enable = "sse4.2")]
fn single_stream(mut state: u32, data: &[u8]) -> u32 {
let mut rest = data;
while let Some((chunk, tail)) = split_chunk(rest) {
let word = u64::from_le_bytes(*chunk);
state = _mm_crc32_u64(u64::from(state), word) as u32;
rest = tail;
}
for &byte in rest {
state = _mm_crc32_u8(state, byte);
}
state
}
#[target_feature(enable = "sse4.2")]
fn interleaved(state: u32, data: &[u8]) -> u32 {
let third = (data.len() / 3) & !7;
if third == 0 {
return single_stream(state, data);
}
let (a, rest) = data.split_at(third);
let (b, rest) = rest.split_at(third);
let (c_head, c_tail) = rest.split_at(third);
let mut state_a = state;
let mut state_b = 0u32;
let mut state_c = 0u32;
let mut ra = a;
let mut rb = b;
let mut rc = c_head;
while let (Some((chunk_a, tail_a)), Some((chunk_b, tail_b)), Some((chunk_c, tail_c))) =
(split_chunk(ra), split_chunk(rb), split_chunk(rc))
{
let word_a = u64::from_le_bytes(*chunk_a);
let word_b = u64::from_le_bytes(*chunk_b);
let word_c = u64::from_le_bytes(*chunk_c);
state_a = _mm_crc32_u64(u64::from(state_a), word_a) as u32;
state_b = _mm_crc32_u64(u64::from(state_b), word_b) as u32;
state_c = _mm_crc32_u64(u64::from(state_c), word_c) as u32;
ra = tail_a;
rb = tail_b;
rc = tail_c;
}
let combined = combine(state_a, state_b, third as u64);
let combined = combine(combined, state_c, third as u64);
single_stream(combined, c_tail)
}
#[cfg(all(test, target_arch = "x86_64"))]
mod tests {
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
use super::*;
use crate::crc32c::scalar::update_state_scalar;
use proptest::prelude::*;
fn with_hw(body: impl FnOnce()) {
if is_available() {
body();
} else {
eprintln!("skipping: SSE4.2 not detected on this CPU");
}
}
unsafe fn hw(state: u32, data: &[u8]) -> u32 {
unsafe { update_state_hw(state, data) }
}
#[test]
fn matches_scalar_on_the_published_check_value() {
with_hw(|| {
let expected = update_state_scalar(u32::MAX, b"123456789");
assert_eq!(unsafe { hw(u32::MAX, b"123456789") }, expected);
});
}
#[test]
fn matches_scalar_exhaustively_for_every_length_up_to_256() {
with_hw(|| {
let data: Vec<u8> = (0..256u32)
.map(|i| (i.wrapping_mul(97) & 0xFF) as u8)
.collect();
for len in 0..=data.len() {
let slice = &data[..len];
let expected = update_state_scalar(0xACE1_5A5A, slice);
assert_eq!(unsafe { hw(0xACE1_5A5A, slice) }, expected, "len {len}");
}
});
}
#[test]
fn matches_scalar_around_the_three_way_threshold_boundary() {
with_hw(|| {
for delta in -8i64..=8 {
let len = (THREE_WAY_THRESHOLD as i64 + delta).max(0) as usize;
let data: Vec<u8> = (0..len)
.map(|i| (i.wrapping_mul(131) & 0xFF) as u8)
.collect();
let expected = update_state_scalar(0x1357_9BDF, &data);
assert_eq!(unsafe { hw(0x1357_9BDF, &data) }, expected, "len {len}");
}
});
}
#[test]
fn matches_scalar_at_every_chunk_multiple_near_the_threshold() {
with_hw(|| {
for len in (THREE_WAY_THRESHOLD - 64)..=(THREE_WAY_THRESHOLD + 64) {
let data: Vec<u8> = (0..len)
.map(|i| (i.wrapping_mul(53) & 0xFF) as u8)
.collect();
let expected = update_state_scalar(0x2222_3333, &data);
assert_eq!(unsafe { hw(0x2222_3333, &data) }, expected, "len {len}");
}
});
}
proptest! {
#![proptest_config(ProptestConfig::with_cases(64))]
#[test]
fn matches_scalar_over_arbitrary_buffers_and_offsets(
seed in any::<u32>(),
len in 0usize..=1_048_576,
offset in 0usize..8,
initial in any::<u32>(),
) {
if !is_available() {
return Ok(());
}
let mut x = seed | 1;
let backing: Vec<u8> = (0..len + offset)
.map(|_| {
x = x.wrapping_mul(1_664_525).wrapping_add(1_013_904_223);
(x >> 24) as u8
})
.collect();
let slice = &backing[offset..];
let expected = update_state_scalar(initial, slice);
let actual = unsafe { hw(initial, slice) };
prop_assert_eq!(actual, expected);
}
}
}