#[cfg(not(feature = "std"))]
use alloc::vec::Vec;
#[cfg(feature = "std")]
use std::vec::Vec;
use crate::error::DecodeError;
mod scalar;
#[cfg(any(target_arch = "x86_64", target_arch = "aarch64"))]
pub(crate) mod shuffle;
#[cfg(target_arch = "x86_64")]
mod avx2;
#[cfg(target_arch = "aarch64")]
mod neon;
#[cfg(target_arch = "x86_64")]
mod sse2;
impl_dispatch_encode!(
dispatch_encode,
u16,
avx2::encode_into,
sse2::encode_into,
neon::encode_into,
scalar::encode_into
);
impl_dispatch_decode!(
dispatch_decode,
u16,
avx2::decode_into,
sse2::decode_into,
neon::decode_into,
scalar::decode_into
);
pub struct Svb16;
impl Svb16 {
pub fn encode(&self, values: &[u16]) -> Vec<u8> {
let mut out = Vec::new();
dispatch_encode(values, &mut out);
out
}
pub fn encode_into(&self, values: &[u16], out: &mut Vec<u8>) {
dispatch_encode(values, out);
}
pub fn decode(&self, data: &[u8], n: usize) -> Result<Vec<u16>, DecodeError> {
let mut out = Vec::with_capacity(n);
dispatch_decode(data, n, &mut out)?;
Ok(out)
}
pub fn decode_into(
&self,
data: &[u8],
n: usize,
out: &mut Vec<u16>,
) -> Result<(), DecodeError> {
dispatch_decode(data, n, out)
}
}
impl crate::coder::Coder for Svb16 {
type Elem = u16;
fn encode_into(&self, values: &[u16], out: &mut Vec<u8>) {
dispatch_encode(values, out);
}
fn decode_into(
&self,
data: &[u8],
n: usize,
out: &mut Vec<Self::Elem>,
) -> Result<(), DecodeError> {
dispatch_decode(data, n, out)
}
fn encoded_data_len(&self, ctrl: &[u8], n: usize) -> usize {
scalar::encoded_data_len(ctrl, n)
}
}
#[cfg(test)]
mod cross_path {
#[cfg(not(feature = "std"))]
use alloc::vec::Vec;
#[cfg(feature = "std")]
use std::vec::Vec;
use super::scalar;
fn decode_scalar(data: &[u8], n: usize) -> Vec<u16> {
let mut out = Vec::new();
scalar::decode_into(data, n, &mut out).unwrap();
out
}
#[cfg(all(target_arch = "x86_64", feature = "std"))]
mod x86 {
use super::super::{avx2, scalar, sse2};
use super::*;
fn encode(values: &[u16]) -> Vec<u8> {
let mut v = Vec::new();
scalar::encode_into(values, &mut v);
v
}
fn ssse3_decode(data: &[u8], n: usize) -> Option<Vec<u16>> {
if !is_x86_feature_detected!("ssse3") {
return None;
}
let mut out = Vec::new();
unsafe { sse2::decode_into(data, n, &mut out).unwrap() };
Some(out)
}
fn avx2_decode(data: &[u8], n: usize) -> Option<Vec<u16>> {
if !is_x86_feature_detected!("avx2") {
return None;
}
let mut out = Vec::new();
unsafe { avx2::decode_into(data, n, &mut out).unwrap() };
Some(out)
}
fn check_all(values: &[u16]) {
let n = values.len();
let enc = encode(values);
let expected = decode_scalar(&enc, n);
if let Some(got) = ssse3_decode(&enc, n) {
assert_eq!(expected, got, "SSSE3 mismatch n={n} values={values:?}");
}
if let Some(got) = avx2_decode(&enc, n) {
assert_eq!(expected, got, "AVX2 mismatch n={n} values={values:?}");
}
}
fn ssse3_encode(values: &[u16]) -> Option<Vec<u8>> {
if !is_x86_feature_detected!("ssse3") {
return None;
}
let mut out = Vec::new();
unsafe { sse2::encode_into(values, &mut out) };
Some(out)
}
fn check_encode(values: &[u16]) {
let expected = encode(values);
if let Some(got) = ssse3_encode(values) {
assert_eq!(
expected,
got,
"SSSE3 encode mismatch n={} values={values:?}",
values.len()
);
}
if let Some(got) = avx2_encode(values) {
assert_eq!(
expected,
got,
"AVX2 encode mismatch n={} values={values:?}",
values.len()
);
}
}
fn avx2_encode(values: &[u16]) -> Option<Vec<u8>> {
if !is_x86_feature_detected!("avx2") {
return None;
}
let mut out = Vec::new();
unsafe { avx2::encode_into(values, &mut out) };
Some(out)
}
const SVB0: &[u8] = include_bytes!("../../tests/vectors/parity_00_02885.svb16");
const SVB1: &[u8] = include_bytes!("../../tests/vectors/parity_01_02915.svb16");
const SVB2: &[u8] = include_bytes!("../../tests/vectors/parity_02_02949.svb16");
#[test]
fn pod5_parity_ssse3() {
for &(data, n) in &[(SVB0, 2885usize), (SVB1, 2915), (SVB2, 2949)] {
if let Some(got) = ssse3_decode(data, n) {
assert_eq!(decode_scalar(data, n), got, "SSSE3 n={n}");
}
}
}
#[test]
fn pod5_parity_avx2() {
for &(data, n) in &[(SVB0, 2885usize), (SVB1, 2915), (SVB2, 2949)] {
if let Some(got) = avx2_decode(data, n) {
assert_eq!(decode_scalar(data, n), got, "AVX2 n={n}");
}
}
}
#[test]
fn all_ctrl_byte_values() {
for ctrl in 0u8..=255 {
let values: Vec<u16> = (0..8)
.map(|k| {
if (ctrl >> k) & 1 == 1 {
300 + k as u16 } else {
k as u16 }
})
.collect();
check_all(&values);
}
}
#[test]
fn ssse3_all_tail_lengths() {
if ssse3_decode(&encode(&[0u16]), 1).is_none() {
return;
}
let pool: Vec<u16> = (0..20)
.map(|i| if i % 3 == 0 { 300 + i } else { i })
.collect();
for n in 0..=20usize {
let values = &pool[..n];
let enc = encode(values);
let expected = decode_scalar(&enc, n);
let got = ssse3_decode(&enc, n).unwrap();
assert_eq!(expected, got, "SSSE3 tail n={n}");
}
}
#[test]
fn avx2_all_tail_lengths() {
if avx2_decode(&encode(&[0u16]), 1).is_none() {
return;
}
let pool: Vec<u16> = (0..33)
.map(|i| if i % 3 == 0 { 300 + i } else { i })
.collect();
for n in 0..=33usize {
let values = &pool[..n];
let enc = encode(values);
let expected = decode_scalar(&enc, n);
let got = avx2_decode(&enc, n).unwrap();
assert_eq!(expected, got, "AVX2 tail n={n}");
}
}
#[test]
fn all_one_byte_values() {
let values: Vec<u16> = (0..=255).collect();
check_all(&values);
}
#[test]
fn all_two_byte_values() {
let values: Vec<u16> = (256..512).collect();
check_all(&values);
}
#[test]
fn alternating_one_and_two_byte() {
let values: Vec<u16> = (0..64)
.map(|i| if i % 2 == 0 { i as u16 } else { 300 + i })
.collect();
check_all(&values);
}
#[test]
fn ssse3_16byte_boundary_guard() {
let block1: Vec<u16> = (1000..1008).collect(); let block2: Vec<u16> = vec![500, 1, 2, 3, 4, 5, 6, 7]; let values: Vec<u16> = block1.into_iter().chain(block2).collect();
check_all(&values);
}
#[test]
fn avx2_32byte_boundary_guard() {
let blocks12: Vec<u16> = (1000..1016).collect();
let block3: Vec<u16> = vec![500, 1, 2, 3, 4, 5, 6, 7];
let values: Vec<u16> = blocks12.into_iter().chain(block3).collect();
check_all(&values);
}
#[test]
fn values_at_type_boundaries() {
let values: Vec<u16> = vec![0, 255, 256, u16::MAX]
.into_iter()
.cycle()
.take(32)
.collect();
check_all(&values);
}
#[test]
fn large_input() {
let values: Vec<u16> = (0..10_000u16)
.map(|i| if i % 7 < 3 { i % 256 } else { 256 + (i % 1000) })
.collect();
check_all(&values);
}
#[test]
fn empty_and_single() {
check_all(&[]);
check_all(&[0]);
check_all(&[255]);
check_all(&[256]);
check_all(&[u16::MAX]);
}
#[test]
fn ssse3_encode_all_ctrl_byte_values() {
for ctrl in 0u8..=255 {
let values: Vec<u16> = (0..8)
.map(|k| {
if (ctrl >> k) & 1 == 1 {
300 + k as u16
} else {
k as u16
}
})
.collect();
check_encode(&values);
}
}
#[test]
fn ssse3_encode_all_tail_lengths() {
if ssse3_encode(&[0u16]).is_none() {
return;
}
let pool: Vec<u16> = (0..20)
.map(|i| if i % 3 == 0 { 300 + i } else { i })
.collect();
for n in 0..=20usize {
check_encode(&pool[..n]);
}
}
#[test]
fn ssse3_encode_roundtrip() {
if ssse3_encode(&[0u16]).is_none() {
return;
}
let values: Vec<u16> = (0..100)
.map(|i| if i % 2 == 0 { i as u16 } else { 300 + i })
.collect();
let enc = ssse3_encode(&values).unwrap();
let got = decode_scalar(&enc, values.len());
assert_eq!(values, got);
}
#[test]
fn ssse3_encode_all_one_byte() {
let values: Vec<u16> = (0..=255).collect();
check_encode(&values);
}
#[test]
fn ssse3_encode_all_two_byte() {
let values: Vec<u16> = (256..512).collect();
check_encode(&values);
}
#[test]
fn ssse3_encode_large_input() {
let values: Vec<u16> = (0..10_000u16)
.map(|i| if i % 7 < 3 { i % 256 } else { 256 + (i % 1000) })
.collect();
check_encode(&values);
}
#[test]
fn ssse3_encode_boundary_values() {
let values: Vec<u16> = [0u16, 255, 256, u16::MAX]
.iter()
.copied()
.cycle()
.take(32)
.collect();
check_encode(&values);
}
#[test]
fn avx2_encode_all_tail_lengths() {
if avx2_encode(&[0u16]).is_none() {
return;
}
let pool: Vec<u16> = (0..33)
.map(|i| if i % 3 == 0 { 300 + i } else { i })
.collect();
for n in 0..=33usize {
check_encode(&pool[..n]);
}
}
#[test]
fn avx2_encode_all_ctrl_byte_values() {
for ctrl in 0u8..=255 {
let values: Vec<u16> = (0..16)
.map(|k| {
if (ctrl >> (k % 8)) & 1 == 1 {
300 + k as u16
} else {
k as u16
}
})
.collect();
check_encode(&values);
}
}
#[test]
fn avx2_encode_roundtrip() {
if avx2_encode(&[0u16]).is_none() {
return;
}
let values: Vec<u16> = (0..200)
.map(|i| if i % 2 == 0 { i as u16 } else { 300 + i })
.collect();
let enc = avx2_encode(&values).unwrap();
let got = decode_scalar(&enc, values.len());
assert_eq!(values, got, "AVX2 encode roundtrip failed");
}
}
#[cfg(target_arch = "aarch64")]
mod arm {
use super::super::{neon, scalar};
use super::*;
#[cfg(not(feature = "std"))]
use alloc::vec;
fn encode(values: &[u16]) -> Vec<u8> {
let mut v = Vec::new();
scalar::encode_into(values, &mut v);
v
}
fn neon_decode(data: &[u8], n: usize) -> Vec<u16> {
let mut out = Vec::new();
unsafe { neon::decode_into(data, n, &mut out).unwrap() };
out
}
fn neon_encode(values: &[u16]) -> Vec<u8> {
let mut out = Vec::new();
unsafe { neon::encode_into(values, &mut out) };
out
}
fn check(values: &[u16]) {
let n = values.len();
let enc = encode(values);
let expected = decode_scalar(&enc, n);
let got = neon_decode(&enc, n);
assert_eq!(expected, got, "NEON decode n={n}");
}
fn check_encode(values: &[u16]) {
let expected = encode(values);
let got = neon_encode(values);
assert_eq!(
expected,
got,
"NEON encode mismatch n={} values={values:?}",
values.len()
);
}
const SVB0: &[u8] = include_bytes!("../../tests/vectors/parity_00_02885.svb16");
const SVB1: &[u8] = include_bytes!("../../tests/vectors/parity_01_02915.svb16");
const SVB2: &[u8] = include_bytes!("../../tests/vectors/parity_02_02949.svb16");
#[test]
fn pod5_parity_neon() {
for &(data, n) in &[(SVB0, 2885usize), (SVB1, 2915), (SVB2, 2949)] {
assert_eq!(decode_scalar(data, n), neon_decode(data, n), "n={n}");
}
}
#[test]
fn all_ctrl_byte_values() {
for ctrl in 0u8..=255 {
let values: Vec<u16> = (0..8)
.map(|k| {
if (ctrl >> k) & 1 == 1 {
300 + k as u16
} else {
k as u16
}
})
.collect();
check(&values);
}
}
#[test]
fn all_tail_lengths() {
let pool: Vec<u16> = (0..20)
.map(|i| if i % 3 == 0 { 300 + i } else { i })
.collect();
for n in 0..=20usize {
let enc = encode(&pool[..n]);
assert_eq!(decode_scalar(&enc, n), neon_decode(&enc, n), "tail n={n}");
}
}
#[test]
fn all_one_byte_values() {
check(&(0u16..=255).collect::<Vec<_>>());
}
#[test]
fn all_two_byte_values() {
check(&(256u16..512).collect::<Vec<_>>());
}
#[test]
fn neon_16byte_boundary_guard() {
let block1: Vec<u16> = (1000..1008).collect();
let block2: Vec<u16> = vec![500, 1, 2, 3, 4, 5, 6, 7];
check(&block1.into_iter().chain(block2).collect::<Vec<_>>());
}
#[test]
fn large_input() {
let values: Vec<u16> = (0..10_000u16)
.map(|i| if i % 7 < 3 { i % 256 } else { 256 + (i % 1000) })
.collect();
check(&values);
}
#[test]
fn neon_encode_all_ctrl_byte_values() {
for ctrl in 0u8..=255 {
let values: Vec<u16> = (0..8)
.map(|k| {
if (ctrl >> k) & 1 == 1 {
300 + k as u16
} else {
k as u16
}
})
.collect();
check_encode(&values);
}
}
#[test]
fn neon_encode_all_tail_lengths() {
let pool: Vec<u16> = (0..20)
.map(|i| if i % 3 == 0 { 300 + i } else { i })
.collect();
for n in 0..=20usize {
check_encode(&pool[..n]);
}
}
#[test]
fn neon_encode_roundtrip() {
let values: Vec<u16> = (0..100)
.map(|i| if i % 2 == 0 { i as u16 } else { 300 + i })
.collect();
let enc = neon_encode(&values);
let got = decode_scalar(&enc, values.len());
assert_eq!(values, got, "NEON encode roundtrip failed");
}
#[test]
fn neon_encode_all_one_byte() {
check_encode(&(0u16..=255).collect::<Vec<_>>());
}
#[test]
fn neon_encode_all_two_byte() {
check_encode(&(256u16..512).collect::<Vec<_>>());
}
#[test]
fn neon_encode_boundary_values() {
let values: Vec<u16> = [0u16, 255, 256, u16::MAX]
.iter()
.copied()
.cycle()
.take(32)
.collect();
check_encode(&values);
}
}
}