pub mod arch;
mod sealed {
pub trait Sealed {}
impl<T> Sealed for &T where T: Sealed + ?Sized {}
impl Sealed for u8 {}
impl Sealed for [u8] {}
impl<const N: usize> Sealed for [u8; N] {}
}
macro_rules! tail_find_fixed {
($name:ident, $($needle:ident),+) => {
#[cfg_attr(not(tarpaulin), inline(always))]
#[allow(clippy::too_many_arguments)]
fn $name(tail: &[u8], $($needle: u8),+) -> Option<usize> {
for (idx, &byte) in tail.iter().enumerate() {
if false $(|| byte == $needle)+ {
return Some(idx);
}
}
None
}
};
}
tail_find_fixed!(tail_find1, a);
tail_find_fixed!(tail_find2, a, b);
tail_find_fixed!(tail_find3, a, b, c);
tail_find_fixed!(tail_find4, a, b, c, d);
tail_find_fixed!(tail_find5, a, b, c, d, e);
tail_find_fixed!(tail_find6, a, b, c, d, e, f);
tail_find_fixed!(tail_find7, a, b, c, d, e, f, g);
tail_find_fixed!(tail_find8, a, b, c, d, e, f, g, h);
macro_rules! prefix_len_fixed {
($name:ident, $($needle:ident),+) => {
#[cfg_attr(not(tarpaulin), inline(always))]
#[allow(clippy::too_many_arguments)]
fn $name(input: &[u8], $($needle: u8),+) -> usize {
for (idx, &byte) in input.iter().enumerate() {
if !(false $(|| byte == $needle)+) {
return idx;
}
}
input.len()
}
};
}
prefix_len_fixed!(prefix_len1, a);
prefix_len_fixed!(prefix_len2, a, b);
prefix_len_fixed!(prefix_len3, a, b, c);
prefix_len_fixed!(prefix_len4, a, b, c, d);
prefix_len_fixed!(prefix_len5, a, b, c, d, e);
prefix_len_fixed!(prefix_len6, a, b, c, d, e, f);
prefix_len_fixed!(prefix_len7, a, b, c, d, e, f, g);
prefix_len_fixed!(prefix_len8, a, b, c, d, e, f, g, h);
pub trait Needles: sealed::Sealed {
fn needle_count(&self) -> usize;
#[cfg_attr(not(tarpaulin), inline(always))]
fn is_empty(&self) -> bool {
self.needle_count() == 0
}
fn tail_find(&self, tail: &[u8]) -> Option<usize>;
fn prefix_len(&self, input: &[u8]) -> usize;
#[cfg(target_arch = "aarch64")]
fn eq_any_mask_neon(
&self,
chunk: core::arch::aarch64::uint8x16_t,
) -> core::arch::aarch64::uint8x16_t;
#[cfg(target_arch = "x86")]
fn eq_any_mask_sse2(&self, chunk: core::arch::x86::__m128i) -> core::arch::x86::__m128i;
#[cfg(target_arch = "x86_64")]
fn eq_any_mask_sse2(&self, chunk: core::arch::x86_64::__m128i) -> core::arch::x86_64::__m128i;
#[cfg(target_arch = "x86_64")]
fn eq_any_mask_avx2(&self, chunk: core::arch::x86_64::__m256i) -> core::arch::x86_64::__m256i;
#[cfg(target_arch = "x86_64")]
fn eq_any_mask_avx512(&self, chunk: core::arch::x86_64::__m512i) -> u64;
#[cfg(all(target_arch = "wasm32", target_feature = "simd128"))]
fn eq_any_mask_simd128(&self, chunk: core::arch::wasm32::v128) -> core::arch::wasm32::v128;
}
impl<T> Needles for &T
where
T: Needles + ?Sized,
{
#[cfg_attr(not(tarpaulin), inline(always))]
fn needle_count(&self) -> usize {
(**self).needle_count()
}
#[cfg_attr(not(tarpaulin), inline(always))]
fn tail_find(&self, tail: &[u8]) -> Option<usize> {
(**self).tail_find(tail)
}
#[cfg_attr(not(tarpaulin), inline(always))]
fn prefix_len(&self, input: &[u8]) -> usize {
(**self).prefix_len(input)
}
#[cfg(target_arch = "aarch64")]
#[cfg_attr(not(tarpaulin), inline(always))]
fn eq_any_mask_neon(
&self,
chunk: core::arch::aarch64::uint8x16_t,
) -> core::arch::aarch64::uint8x16_t {
(**self).eq_any_mask_neon(chunk)
}
#[cfg(target_arch = "x86")]
#[cfg_attr(not(tarpaulin), inline(always))]
fn eq_any_mask_sse2(&self, chunk: core::arch::x86::__m128i) -> core::arch::x86::__m128i {
(**self).eq_any_mask_sse2(chunk)
}
#[cfg(target_arch = "x86_64")]
#[cfg_attr(not(tarpaulin), inline(always))]
fn eq_any_mask_sse2(&self, chunk: core::arch::x86_64::__m128i) -> core::arch::x86_64::__m128i {
(**self).eq_any_mask_sse2(chunk)
}
#[cfg(target_arch = "x86_64")]
#[cfg_attr(not(tarpaulin), inline(always))]
fn eq_any_mask_avx2(&self, chunk: core::arch::x86_64::__m256i) -> core::arch::x86_64::__m256i {
(**self).eq_any_mask_avx2(chunk)
}
#[cfg(target_arch = "x86_64")]
#[cfg_attr(not(tarpaulin), inline(always))]
fn eq_any_mask_avx512(&self, chunk: core::arch::x86_64::__m512i) -> u64 {
(**self).eq_any_mask_avx512(chunk)
}
#[cfg(all(target_arch = "wasm32", target_feature = "simd128"))]
#[cfg_attr(not(tarpaulin), inline(always))]
fn eq_any_mask_simd128(&self, chunk: core::arch::wasm32::v128) -> core::arch::wasm32::v128 {
(**self).eq_any_mask_simd128(chunk)
}
}
impl Needles for u8 {
#[cfg_attr(not(tarpaulin), inline(always))]
fn needle_count(&self) -> usize {
1
}
#[cfg_attr(not(tarpaulin), inline(always))]
fn tail_find(&self, tail: &[u8]) -> Option<usize> {
tail_find1(tail, *self)
}
#[cfg_attr(not(tarpaulin), inline(always))]
fn prefix_len(&self, input: &[u8]) -> usize {
prefix_len1(input, *self)
}
#[cfg(target_arch = "aarch64")]
#[cfg_attr(not(tarpaulin), inline(always))]
fn eq_any_mask_neon(
&self,
chunk: core::arch::aarch64::uint8x16_t,
) -> core::arch::aarch64::uint8x16_t {
let target = unsafe { core::arch::aarch64::vdupq_n_u8(*self) };
unsafe { core::arch::aarch64::vceqq_u8(chunk, target) }
}
#[cfg(target_arch = "x86")]
#[cfg_attr(not(tarpaulin), inline(always))]
fn eq_any_mask_sse2(&self, chunk: core::arch::x86::__m128i) -> core::arch::x86::__m128i {
unsafe {
use core::arch::x86::*;
_mm_cmpeq_epi8(chunk, _mm_set1_epi8(*self as i8))
}
}
#[cfg(target_arch = "x86_64")]
#[cfg_attr(not(tarpaulin), inline(always))]
fn eq_any_mask_sse2(&self, chunk: core::arch::x86_64::__m128i) -> core::arch::x86_64::__m128i {
unsafe {
use core::arch::x86_64::*;
_mm_cmpeq_epi8(chunk, _mm_set1_epi8(*self as i8))
}
}
#[cfg(target_arch = "x86_64")]
#[cfg_attr(not(tarpaulin), inline(always))]
fn eq_any_mask_avx2(&self, chunk: core::arch::x86_64::__m256i) -> core::arch::x86_64::__m256i {
unsafe {
use core::arch::x86_64::*;
_mm256_cmpeq_epi8(chunk, _mm256_set1_epi8(*self as i8))
}
}
#[cfg(target_arch = "x86_64")]
#[cfg_attr(not(tarpaulin), inline(always))]
fn eq_any_mask_avx512(&self, chunk: core::arch::x86_64::__m512i) -> u64 {
unsafe {
use core::arch::x86_64::*;
_mm512_cmpeq_epi8_mask(chunk, _mm512_set1_epi8(*self as i8))
}
}
#[cfg(all(target_arch = "wasm32", target_feature = "simd128"))]
#[cfg_attr(not(tarpaulin), inline(always))]
fn eq_any_mask_simd128(&self, chunk: core::arch::wasm32::v128) -> core::arch::wasm32::v128 {
use core::arch::wasm32::*;
i8x16_eq(chunk, i8x16_splat(*self as i8))
}
}
impl Needles for [u8] {
#[cfg_attr(not(tarpaulin), inline(always))]
fn needle_count(&self) -> usize {
self.len()
}
#[cfg_attr(not(tarpaulin), inline(always))]
fn tail_find(&self, tail: &[u8]) -> Option<usize> {
match self {
[] => None,
[a] => tail_find1(tail, *a),
[a, b] => tail_find2(tail, *a, *b),
[a, b, c] => tail_find3(tail, *a, *b, *c),
[a, b, c, d] => tail_find4(tail, *a, *b, *c, *d),
[a, b, c, d, e] => tail_find5(tail, *a, *b, *c, *d, *e),
[a, b, c, d, e, f] => tail_find6(tail, *a, *b, *c, *d, *e, *f),
[a, b, c, d, e, f, g] => tail_find7(tail, *a, *b, *c, *d, *e, *f, *g),
[a, b, c, d, e, f, g, h] => tail_find8(tail, *a, *b, *c, *d, *e, *f, *g, *h),
_ => tail.iter().position(|byte| self.contains(byte)),
}
}
#[cfg_attr(not(tarpaulin), inline(always))]
fn prefix_len(&self, input: &[u8]) -> usize {
match self {
[] => 0,
[a] => prefix_len1(input, *a),
[a, b] => prefix_len2(input, *a, *b),
[a, b, c] => prefix_len3(input, *a, *b, *c),
[a, b, c, d] => prefix_len4(input, *a, *b, *c, *d),
[a, b, c, d, e] => prefix_len5(input, *a, *b, *c, *d, *e),
[a, b, c, d, e, f] => prefix_len6(input, *a, *b, *c, *d, *e, *f),
[a, b, c, d, e, f, g] => prefix_len7(input, *a, *b, *c, *d, *e, *f, *g),
[a, b, c, d, e, f, g, h] => prefix_len8(input, *a, *b, *c, *d, *e, *f, *g, *h),
_ => input
.iter()
.position(|byte| !self.contains(byte))
.unwrap_or(input.len()),
}
}
#[cfg(target_arch = "aarch64")]
#[cfg_attr(not(tarpaulin), inline(always))]
fn eq_any_mask_neon(
&self,
chunk: core::arch::aarch64::uint8x16_t,
) -> core::arch::aarch64::uint8x16_t {
match self {
[] => arch::aarch64::eq_any_mask_const(chunk, []),
[a] => arch::aarch64::eq_any_mask_const(chunk, [*a]),
[a, b] => arch::aarch64::eq_any_mask_const(chunk, [*a, *b]),
[a, b, c] => arch::aarch64::eq_any_mask_const(chunk, [*a, *b, *c]),
[a, b, c, d] => arch::aarch64::eq_any_mask_const(chunk, [*a, *b, *c, *d]),
[a, b, c, d, e] => arch::aarch64::eq_any_mask_const(chunk, [*a, *b, *c, *d, *e]),
[a, b, c, d, e, f] => arch::aarch64::eq_any_mask_const(chunk, [*a, *b, *c, *d, *e, *f]),
[a, b, c, d, e, f, g] => {
arch::aarch64::eq_any_mask_const(chunk, [*a, *b, *c, *d, *e, *f, *g])
}
[a, b, c, d, e, f, g, h] => {
arch::aarch64::eq_any_mask_const(chunk, [*a, *b, *c, *d, *e, *f, *g, *h])
}
_ => arch::aarch64::eq_any_mask_dynamic(chunk, self),
}
}
#[cfg(target_arch = "x86")]
#[cfg_attr(not(tarpaulin), inline(always))]
fn eq_any_mask_sse2(&self, chunk: core::arch::x86::__m128i) -> core::arch::x86::__m128i {
match self {
[] => unsafe { core::arch::x86::_mm_setzero_si128() },
[a] => unsafe { arch::x86::eq_any_mask_const_sse2(chunk, [*a]) },
[a, b] => unsafe { arch::x86::eq_any_mask_const_sse2(chunk, [*a, *b]) },
[a, b, c] => unsafe { arch::x86::eq_any_mask_const_sse2(chunk, [*a, *b, *c]) },
[a, b, c, d] => unsafe { arch::x86::eq_any_mask_const_sse2(chunk, [*a, *b, *c, *d]) },
[a, b, c, d, e] => unsafe { arch::x86::eq_any_mask_const_sse2(chunk, [*a, *b, *c, *d, *e]) },
[a, b, c, d, e, f] => unsafe {
arch::x86::eq_any_mask_const_sse2(chunk, [*a, *b, *c, *d, *e, *f])
},
[a, b, c, d, e, f, g] => unsafe {
arch::x86::eq_any_mask_const_sse2(chunk, [*a, *b, *c, *d, *e, *f, *g])
},
[a, b, c, d, e, f, g, h] => unsafe {
arch::x86::eq_any_mask_const_sse2(chunk, [*a, *b, *c, *d, *e, *f, *g, *h])
},
_ => unsafe { arch::x86::eq_any_mask_dynamic_sse2(chunk, self) },
}
}
#[cfg(target_arch = "x86_64")]
#[cfg_attr(not(tarpaulin), inline(always))]
fn eq_any_mask_sse2(&self, chunk: core::arch::x86_64::__m128i) -> core::arch::x86_64::__m128i {
match self {
[] => unsafe { core::arch::x86_64::_mm_setzero_si128() },
[a] => unsafe { arch::x86::eq_any_mask_const_sse2(chunk, [*a]) },
[a, b] => unsafe { arch::x86::eq_any_mask_const_sse2(chunk, [*a, *b]) },
[a, b, c] => unsafe { arch::x86::eq_any_mask_const_sse2(chunk, [*a, *b, *c]) },
[a, b, c, d] => unsafe { arch::x86::eq_any_mask_const_sse2(chunk, [*a, *b, *c, *d]) },
[a, b, c, d, e] => unsafe { arch::x86::eq_any_mask_const_sse2(chunk, [*a, *b, *c, *d, *e]) },
[a, b, c, d, e, f] => unsafe {
arch::x86::eq_any_mask_const_sse2(chunk, [*a, *b, *c, *d, *e, *f])
},
[a, b, c, d, e, f, g] => unsafe {
arch::x86::eq_any_mask_const_sse2(chunk, [*a, *b, *c, *d, *e, *f, *g])
},
[a, b, c, d, e, f, g, h] => unsafe {
arch::x86::eq_any_mask_const_sse2(chunk, [*a, *b, *c, *d, *e, *f, *g, *h])
},
_ => unsafe { arch::x86::eq_any_mask_dynamic_sse2(chunk, self) },
}
}
#[cfg(target_arch = "x86_64")]
#[cfg_attr(not(tarpaulin), inline(always))]
fn eq_any_mask_avx2(&self, chunk: core::arch::x86_64::__m256i) -> core::arch::x86_64::__m256i {
match self {
[] => unsafe { core::arch::x86_64::_mm256_setzero_si256() },
[a] => unsafe { arch::x86::eq_any_mask_const_avx2(chunk, [*a]) },
[a, b] => unsafe { arch::x86::eq_any_mask_const_avx2(chunk, [*a, *b]) },
[a, b, c] => unsafe { arch::x86::eq_any_mask_const_avx2(chunk, [*a, *b, *c]) },
[a, b, c, d] => unsafe { arch::x86::eq_any_mask_const_avx2(chunk, [*a, *b, *c, *d]) },
[a, b, c, d, e] => unsafe { arch::x86::eq_any_mask_const_avx2(chunk, [*a, *b, *c, *d, *e]) },
[a, b, c, d, e, f] => unsafe {
arch::x86::eq_any_mask_const_avx2(chunk, [*a, *b, *c, *d, *e, *f])
},
[a, b, c, d, e, f, g] => unsafe {
arch::x86::eq_any_mask_const_avx2(chunk, [*a, *b, *c, *d, *e, *f, *g])
},
[a, b, c, d, e, f, g, h] => unsafe {
arch::x86::eq_any_mask_const_avx2(chunk, [*a, *b, *c, *d, *e, *f, *g, *h])
},
_ => unsafe { arch::x86::eq_any_mask_dynamic_avx2(chunk, self) },
}
}
#[cfg(target_arch = "x86_64")]
#[cfg_attr(not(tarpaulin), inline(always))]
fn eq_any_mask_avx512(&self, chunk: core::arch::x86_64::__m512i) -> u64 {
match self {
[] => 0u64,
[a] => unsafe { arch::x86::eq_any_mask_const_avx512(chunk, [*a]) },
[a, b] => unsafe { arch::x86::eq_any_mask_const_avx512(chunk, [*a, *b]) },
[a, b, c] => unsafe { arch::x86::eq_any_mask_const_avx512(chunk, [*a, *b, *c]) },
[a, b, c, d] => unsafe { arch::x86::eq_any_mask_const_avx512(chunk, [*a, *b, *c, *d]) },
[a, b, c, d, e] => unsafe {
arch::x86::eq_any_mask_const_avx512(chunk, [*a, *b, *c, *d, *e])
},
[a, b, c, d, e, f] => unsafe {
arch::x86::eq_any_mask_const_avx512(chunk, [*a, *b, *c, *d, *e, *f])
},
[a, b, c, d, e, f, g] => unsafe {
arch::x86::eq_any_mask_const_avx512(chunk, [*a, *b, *c, *d, *e, *f, *g])
},
[a, b, c, d, e, f, g, h] => unsafe {
arch::x86::eq_any_mask_const_avx512(chunk, [*a, *b, *c, *d, *e, *f, *g, *h])
},
_ => unsafe { arch::x86::eq_any_mask_dynamic_avx512(chunk, self) },
}
}
#[cfg(all(target_arch = "wasm32", target_feature = "simd128"))]
#[cfg_attr(not(tarpaulin), inline(always))]
fn eq_any_mask_simd128(&self, chunk: core::arch::wasm32::v128) -> core::arch::wasm32::v128 {
match self {
[] => arch::wasm32::eq_any_mask_const_simd128(chunk, []),
[a] => arch::wasm32::eq_any_mask_const_simd128(chunk, [*a]),
[a, b] => arch::wasm32::eq_any_mask_const_simd128(chunk, [*a, *b]),
[a, b, c] => arch::wasm32::eq_any_mask_const_simd128(chunk, [*a, *b, *c]),
[a, b, c, d] => arch::wasm32::eq_any_mask_const_simd128(chunk, [*a, *b, *c, *d]),
[a, b, c, d, e] => arch::wasm32::eq_any_mask_const_simd128(chunk, [*a, *b, *c, *d, *e]),
[a, b, c, d, e, f] => {
arch::wasm32::eq_any_mask_const_simd128(chunk, [*a, *b, *c, *d, *e, *f])
}
[a, b, c, d, e, f, g] => {
arch::wasm32::eq_any_mask_const_simd128(chunk, [*a, *b, *c, *d, *e, *f, *g])
}
[a, b, c, d, e, f, g, h] => {
arch::wasm32::eq_any_mask_const_simd128(chunk, [*a, *b, *c, *d, *e, *f, *g, *h])
}
_ => arch::wasm32::eq_any_mask_dynamic_simd128(chunk, self),
}
}
}
impl<const N: usize> Needles for [u8; N] {
#[cfg_attr(not(tarpaulin), inline(always))]
fn needle_count(&self) -> usize {
N
}
#[cfg_attr(not(tarpaulin), inline(always))]
fn tail_find(&self, tail: &[u8]) -> Option<usize> {
match self.as_slice() {
[] => None,
[a] => tail_find1(tail, *a),
[a, b] => tail_find2(tail, *a, *b),
[a, b, c] => tail_find3(tail, *a, *b, *c),
[a, b, c, d] => tail_find4(tail, *a, *b, *c, *d),
[a, b, c, d, e] => tail_find5(tail, *a, *b, *c, *d, *e),
[a, b, c, d, e, f] => tail_find6(tail, *a, *b, *c, *d, *e, *f),
[a, b, c, d, e, f, g] => tail_find7(tail, *a, *b, *c, *d, *e, *f, *g),
[a, b, c, d, e, f, g, h] => tail_find8(tail, *a, *b, *c, *d, *e, *f, *g, *h),
_ => tail.iter().position(|byte| self.contains(byte)),
}
}
#[cfg_attr(not(tarpaulin), inline(always))]
fn prefix_len(&self, input: &[u8]) -> usize {
match self.as_slice() {
[] => 0,
[a] => prefix_len1(input, *a),
[a, b] => prefix_len2(input, *a, *b),
[a, b, c] => prefix_len3(input, *a, *b, *c),
[a, b, c, d] => prefix_len4(input, *a, *b, *c, *d),
[a, b, c, d, e] => prefix_len5(input, *a, *b, *c, *d, *e),
[a, b, c, d, e, f] => prefix_len6(input, *a, *b, *c, *d, *e, *f),
[a, b, c, d, e, f, g] => prefix_len7(input, *a, *b, *c, *d, *e, *f, *g),
[a, b, c, d, e, f, g, h] => prefix_len8(input, *a, *b, *c, *d, *e, *f, *g, *h),
_ => input
.iter()
.position(|byte| !self.contains(byte))
.unwrap_or(input.len()),
}
}
#[cfg(target_arch = "aarch64")]
#[cfg_attr(not(tarpaulin), inline(always))]
fn eq_any_mask_neon(
&self,
chunk: core::arch::aarch64::uint8x16_t,
) -> core::arch::aarch64::uint8x16_t {
arch::aarch64::eq_any_mask_const(chunk, *self)
}
#[cfg(target_arch = "x86")]
#[cfg_attr(not(tarpaulin), inline(always))]
fn eq_any_mask_sse2(&self, chunk: core::arch::x86::__m128i) -> core::arch::x86::__m128i {
unsafe { arch::x86::eq_any_mask_const_sse2(chunk, *self) }
}
#[cfg(target_arch = "x86_64")]
#[cfg_attr(not(tarpaulin), inline(always))]
fn eq_any_mask_sse2(&self, chunk: core::arch::x86_64::__m128i) -> core::arch::x86_64::__m128i {
unsafe { arch::x86::eq_any_mask_const_sse2(chunk, *self) }
}
#[cfg(target_arch = "x86_64")]
#[cfg_attr(not(tarpaulin), inline(always))]
fn eq_any_mask_avx2(&self, chunk: core::arch::x86_64::__m256i) -> core::arch::x86_64::__m256i {
unsafe { arch::x86::eq_any_mask_const_avx2(chunk, *self) }
}
#[cfg(target_arch = "x86_64")]
#[cfg_attr(not(tarpaulin), inline(always))]
fn eq_any_mask_avx512(&self, chunk: core::arch::x86_64::__m512i) -> u64 {
unsafe { arch::x86::eq_any_mask_const_avx512(chunk, *self) }
}
#[cfg(all(target_arch = "wasm32", target_feature = "simd128"))]
#[cfg_attr(not(tarpaulin), inline(always))]
fn eq_any_mask_simd128(&self, chunk: core::arch::wasm32::v128) -> core::arch::wasm32::v128 {
arch::wasm32::eq_any_mask_const_simd128(chunk, *self)
}
}