#[must_use]
pub fn find_any_of(haystack: &[u8], needles: &[u8]) -> Option<usize> {
match needles.len() {
0 => None,
1 => memchr::memchr(needles[0], haystack),
2 => memchr::memchr2(needles[0], needles[1], haystack),
3 => memchr::memchr3(needles[0], needles[1], needles[2], haystack),
_ => find_any_of_many(haystack, needles),
}
}
#[must_use]
pub fn clean_prefix_len(haystack: &[u8], needles: &[u8]) -> usize {
find_any_of(haystack, needles).unwrap_or(haystack.len())
}
#[must_use]
pub fn match_prefix_len(haystack: &[u8], needles: &[u8]) -> usize {
if needles.is_empty() {
return 0;
}
if needles.len() == 1 {
let n = needles[0];
let mut i = 0;
while i < haystack.len() && haystack[i] == n {
i += 1;
}
return i;
}
let bm = bitmap_for(needles);
let mut i = 0;
while i < haystack.len() && bm.contains(haystack[i]) {
i += 1;
}
i
}
#[must_use]
pub fn find_byte_in_bitmap(haystack: &[u8], bitmap: &ByteBitmap) -> Option<usize> {
for (i, &b) in haystack.iter().enumerate() {
if bitmap.contains(b) {
return Some(i);
}
}
None
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub struct ByteBitmap {
lanes: [u64; 4],
}
impl ByteBitmap {
#[must_use]
#[inline]
pub fn contains(self, byte: u8) -> bool {
let lane = (byte >> 6) as usize;
let bit = byte & 0x3F;
(self.lanes[lane] >> bit) & 1 == 1
}
}
#[must_use]
pub fn bitmap_for(needles: &[u8]) -> ByteBitmap {
let mut lanes = [0u64; 4];
for &b in needles {
let lane = (b >> 6) as usize;
let bit = b & 0x3F;
lanes[lane] |= 1u64 << bit;
}
ByteBitmap { lanes }
}
fn find_any_of_many(haystack: &[u8], needles: &[u8]) -> Option<usize> {
let bitmap = bitmap_for(needles);
let mut i = 0;
let chunk = 8;
while i + chunk <= haystack.len() {
for j in 0..chunk {
if bitmap.contains(haystack[i + j]) {
return Some(i + j);
}
}
i += chunk;
}
while i < haystack.len() {
if bitmap.contains(haystack[i]) {
return Some(i);
}
i += 1;
}
None
}
#[derive(Debug, Clone, Copy)]
pub struct SimdScanner {
bitmap: ByteBitmap,
#[cfg_attr(not(all(feature = "nightly-simd", noyalib_nightly)), allow(dead_code))]
needles: [u8; 16],
#[cfg_attr(not(all(feature = "nightly-simd", noyalib_nightly)), allow(dead_code))]
needle_count: u8,
}
impl SimdScanner {
#[must_use]
pub fn new(needles: &[u8]) -> Self {
let bitmap = bitmap_for(needles);
let mut compact = [0u8; 16];
let mut count: u8 = 0;
let mut seen = [false; 256];
for &b in needles {
if seen[b as usize] {
continue;
}
seen[b as usize] = true;
if (count as usize) < compact.len() {
compact[count as usize] = b;
count += 1;
}
}
SimdScanner {
bitmap,
needles: compact,
needle_count: count,
}
}
#[must_use]
pub fn find_any(&self, haystack: &[u8]) -> Option<usize> {
#[cfg(all(feature = "nightly-simd", noyalib_nightly))]
{
self.find_any_simd(haystack)
}
#[cfg(not(all(feature = "nightly-simd", noyalib_nightly)))]
{
self.find_any_scalar(haystack)
}
}
fn find_any_scalar(&self, haystack: &[u8]) -> Option<usize> {
for (i, &b) in haystack.iter().enumerate() {
if self.bitmap.contains(b) {
return Some(i);
}
}
None
}
#[cfg(all(feature = "nightly-simd", noyalib_nightly))]
fn find_any_simd(&self, haystack: &[u8]) -> Option<usize> {
use core::simd::cmp::SimdPartialEq;
use core::simd::{Mask, Simd};
const LANES: usize = 32;
let needle_count = self.needle_count as usize;
if needle_count > 8 {
return self.find_any_scalar(haystack);
}
let active = &self.needles[..needle_count];
let mut i = 0;
while i + LANES <= haystack.len() {
let chunk: Simd<u8, LANES> = Simd::from_slice(&haystack[i..i + LANES]);
let mut combined: Mask<i8, LANES> = Mask::splat(false);
for &n in active {
let needle: Simd<u8, LANES> = Simd::splat(n);
combined |= chunk.simd_eq(needle);
}
let bits = combined.to_bitmask();
if bits != 0 {
return Some(i + bits.trailing_zeros() as usize);
}
i += LANES;
}
for (offset, &b) in haystack[i..].iter().enumerate() {
if self.bitmap.contains(b) {
return Some(i + offset);
}
}
None
}
#[must_use]
pub fn structural_bitmask_32(&self, chunk: &[u8; 32]) -> u32 {
#[cfg(all(feature = "nightly-simd", noyalib_nightly))]
{
self.structural_bitmask_32_simd(chunk)
}
#[cfg(not(all(feature = "nightly-simd", noyalib_nightly)))]
{
self.structural_bitmask_32_scalar(chunk)
}
}
fn structural_bitmask_32_scalar(&self, chunk: &[u8; 32]) -> u32 {
let mut mask: u32 = 0;
for (i, &b) in chunk.iter().enumerate() {
mask |= u32::from(self.bitmap.contains(b)) << i;
}
mask
}
#[cfg(all(feature = "nightly-simd", noyalib_nightly))]
fn structural_bitmask_32_simd(&self, chunk: &[u8; 32]) -> u32 {
use core::simd::cmp::SimdPartialEq;
use core::simd::{Mask, Simd};
let needle_count = self.needle_count as usize;
if needle_count == 0 {
return 0;
}
if needle_count > 8 {
return self.structural_bitmask_32_scalar(chunk);
}
let active = &self.needles[..needle_count];
let chunk_v: Simd<u8, 32> = Simd::from_slice(chunk);
let mut combined: Mask<i8, 32> = Mask::splat(false);
for &n in active {
let needle: Simd<u8, 32> = Simd::splat(n);
combined |= chunk_v.simd_eq(needle);
}
combined.to_bitmask() as u32
}
}
pub const BLOCK_PLAIN_NEEDLES: &[u8] = b": \t";
pub const FLOW_PLAIN_NEEDLES: &[u8] = b": \t,[]{}";
pub const LINE_BREAK_NEEDLES: &[u8] = b"\n\r";
#[derive(Debug)]
pub struct StructuralIter<'a> {
scanner: &'a SimdScanner,
haystack: &'a [u8],
cursor: usize,
cached_base: usize,
cached_mask: u32,
}
impl<'a> StructuralIter<'a> {
#[must_use]
pub fn new(scanner: &'a SimdScanner, haystack: &'a [u8]) -> Self {
StructuralIter {
scanner,
haystack,
cursor: 0,
cached_base: 0,
cached_mask: 0,
}
}
}
impl Iterator for StructuralIter<'_> {
type Item = usize;
fn next(&mut self) -> Option<usize> {
if self.cached_mask != 0 {
let bit = self.cached_mask.trailing_zeros() as usize;
self.cached_mask &= self.cached_mask - 1;
return Some(self.cached_base + bit);
}
loop {
if self.cursor >= self.haystack.len() {
return None;
}
let remaining = self.haystack.len() - self.cursor;
if remaining >= 32 {
let mut chunk = [0u8; 32];
chunk.copy_from_slice(&self.haystack[self.cursor..self.cursor + 32]);
let mask = self.scanner.structural_bitmask_32(&chunk);
let chunk_origin = self.cursor;
self.cursor += 32;
if mask != 0 {
let bit = mask.trailing_zeros() as usize;
self.cached_base = chunk_origin;
self.cached_mask = mask & (mask - 1);
return Some(chunk_origin + bit);
}
continue;
}
for offset in 0..remaining {
let b = self.haystack[self.cursor + offset];
if self.scanner.bitmap.contains(b) {
let pos = self.cursor + offset;
self.cursor = pos + 1;
return Some(pos);
}
}
self.cursor = self.haystack.len();
return None;
}
}
}
#[must_use]
pub fn parse_decimal_u64(bytes: &[u8]) -> Option<u64> {
if bytes.is_empty() {
return None;
}
let mut result: u64 = 0;
let mut i = 0;
while i + 8 <= bytes.len() {
let mut arr = [0u8; 8];
arr.copy_from_slice(&bytes[i..i + 8]);
match parse_8_digits(arr) {
Some(eight) => {
result = result.checked_mul(100_000_000)?.checked_add(eight)?;
i += 8;
}
None => return None,
}
}
while i < bytes.len() {
let b = bytes[i];
if !b.is_ascii_digit() {
return None;
}
result = result.checked_mul(10)?.checked_add((b - b'0') as u64)?;
i += 1;
}
Some(result)
}
#[must_use]
pub fn parse_decimal_i64(bytes: &[u8]) -> Option<i64> {
if bytes.is_empty() {
return None;
}
let (negative, digits) = match bytes[0] {
b'-' => (true, &bytes[1..]),
b'+' => (false, &bytes[1..]),
_ => (false, bytes),
};
let abs = parse_decimal_u64(digits)?;
if negative {
if abs == (i64::MAX as u64) + 1 {
Some(i64::MIN)
} else {
i64::try_from(abs).ok().map(|n| -n)
}
} else {
i64::try_from(abs).ok()
}
}
fn parse_8_digits(arr: [u8; 8]) -> Option<u64> {
let chunk = u64::from_be_bytes(arr);
let sub = chunk.wrapping_sub(0x3030_3030_3030_3030);
let above_9 = sub.wrapping_add(0x7676_7676_7676_7676) & 0x8080_8080_8080_8080;
let below_0 = chunk & 0x8080_8080_8080_8080;
if above_9 != 0 || below_0 != 0 {
return None;
}
let high = (sub & 0xFF00_FF00_FF00_FF00) >> 8;
let low = sub & 0x00FF_00FF_00FF_00FF;
let chunk = high.wrapping_mul(10).wrapping_add(low);
let high = (chunk & 0xFFFF_0000_FFFF_0000) >> 16;
let low = chunk & 0x0000_FFFF_0000_FFFF;
let chunk = high.wrapping_mul(100).wrapping_add(low);
let high = chunk >> 32;
let low = chunk & 0xFFFF_FFFF;
Some(high.wrapping_mul(10_000).wrapping_add(low))
}
#[cfg(test)]
#[allow(clippy::byte_char_slices)]
mod tests {
use super::*;
#[test]
fn empty_haystack_returns_none() {
assert_eq!(find_any_of(b"", &[b':']), None);
assert_eq!(find_any_of(b"", &[b':', b',', b'#', b'=']), None);
}
#[test]
fn empty_needles_returns_none() {
assert_eq!(find_any_of(b"hello", &[]), None);
}
#[test]
fn arity_1_finds_first() {
assert_eq!(find_any_of(b"abc:def:ghi", &[b':']), Some(3));
}
#[test]
fn arity_2_finds_first_of_either() {
assert_eq!(find_any_of(b"abc#def\n", &[b':', b'#']), Some(3));
assert_eq!(find_any_of(b"abc\ndef#", &[b':', b'#']), Some(7));
}
#[test]
fn arity_3_finds_first_of_three() {
assert_eq!(find_any_of(b"abc\ndef:ghi", &[b':', b'\n', b'#']), Some(3));
}
#[test]
fn arity_4_uses_swar_path() {
let needles = &[b':', b',', b'=', b'\n'];
assert_eq!(find_any_of(b"abc=def", needles), Some(3));
assert_eq!(find_any_of(b"plain text only", needles), None);
}
#[test]
fn arity_8_finds_correctly() {
let needles = b"[]{}:,#\n";
assert_eq!(find_any_of(b"hello{world", needles), Some(5));
assert_eq!(find_any_of(b"plain text", needles), None);
}
#[test]
fn boundary_at_chunk_edge() {
for pos in 0..16 {
let mut input = vec![b'.'; 16];
input[pos] = b':';
assert_eq!(
find_any_of(&input, &[b':', b',', b'#', b'\n', b'=']),
Some(pos),
"needle at position {pos}",
);
}
}
#[test]
fn long_haystack_finds_at_far_position() {
let mut input = vec![b'.'; 1024];
input[1000] = b':';
assert_eq!(
find_any_of(&input, &[b':', b',', b'\n', b'#', b'=']),
Some(1000),
);
}
#[test]
fn long_haystack_with_no_match_returns_none() {
let input = vec![b'.'; 1024];
assert_eq!(find_any_of(&input, &[b':', b',', b'#', b'\n', b'=']), None);
}
#[test]
fn clean_prefix_len_basic() {
assert_eq!(clean_prefix_len(b"abc:def", &[b':']), 3);
assert_eq!(clean_prefix_len(b"all clean", &[b':']), 9);
assert_eq!(clean_prefix_len(b":start", &[b':']), 0);
}
#[test]
fn bitmap_membership() {
let bm = bitmap_for(&[b'A', b'\n', 0]);
assert!(bm.contains(b'A'));
assert!(bm.contains(b'\n'));
assert!(bm.contains(0));
assert!(!bm.contains(b'B'));
assert!(!bm.contains(255));
}
#[test]
fn bitmap_full_set() {
let needles: Vec<u8> = (0u8..=255).collect();
let bm = bitmap_for(&needles);
for b in 0u8..=255 {
assert!(bm.contains(b), "byte {b}");
}
}
#[test]
fn swar_equivalence_with_scalar() {
let scalar = |haystack: &[u8], needles: &[u8]| -> Option<usize> {
for (i, &b) in haystack.iter().enumerate() {
if needles.contains(&b) {
return Some(i);
}
}
None
};
let needle_sets: &[&[u8]] = &[
b"[]{}:,#\n",
b":,=", b":", b":,#=[]{}\n\t \"'", ];
for &needles in needle_sets {
for length in [0usize, 1, 7, 8, 9, 16, 17, 63, 64, 65, 1023, 1024] {
let mut buf = vec![0u8; length];
for (i, slot) in buf.iter_mut().enumerate() {
*slot = (i as u8).wrapping_add(33); while needles.contains(slot) {
*slot = slot.wrapping_add(1);
}
}
assert_eq!(find_any_of(&buf, needles), scalar(&buf, needles));
for pos in 0..length {
let saved = buf[pos];
buf[pos] = needles[pos % needles.len()];
assert_eq!(
find_any_of(&buf, needles),
scalar(&buf, needles),
"mismatch needles={:?} length={length} pos={pos}",
needles,
);
buf[pos] = saved;
}
}
}
}
#[test]
fn scanner_basic_finds_first_needle() {
let s = SimdScanner::new(b":-{}");
assert_eq!(s.find_any(b"a-b"), Some(1));
assert_eq!(s.find_any(b"abc"), None);
}
#[test]
fn scanner_long_input_match_at_far_position() {
let s = SimdScanner::new(b":,#\n[]{}");
let mut buf = vec![b'.'; 4096];
buf[3000] = b'#';
assert_eq!(s.find_any(&buf), Some(3000));
}
#[test]
fn scanner_matches_baseline_across_needle_sets() {
let baseline = |haystack: &[u8], needles: &[u8]| -> Option<usize> {
for (i, &b) in haystack.iter().enumerate() {
if needles.contains(&b) {
return Some(i);
}
}
None
};
let needle_sets: &[&[u8]] = &[
b":\n",
b":,#\n",
b"[]{}:,#\n",
b"abcdefghij", ];
for &needles in needle_sets {
let s = SimdScanner::new(needles);
for length in [0usize, 1, 31, 32, 33, 64, 127, 128, 1024] {
let mut buf = vec![0u8; length];
for (i, slot) in buf.iter_mut().enumerate() {
*slot = (i as u8).wrapping_add(33);
while needles.contains(slot) {
*slot = slot.wrapping_add(1);
}
}
assert_eq!(
s.find_any(&buf),
baseline(&buf, needles),
"no-needle: needles={needles:?} len={length}"
);
for pos in 0..length {
let saved = buf[pos];
buf[pos] = needles[pos % needles.len()];
assert_eq!(
s.find_any(&buf),
baseline(&buf, needles),
"needles={needles:?} len={length} pos={pos}",
);
buf[pos] = saved;
}
}
}
}
fn baseline_bitmask_32(chunk: &[u8; 32], needles: &[u8]) -> u32 {
let mut m: u32 = 0;
for (i, &b) in chunk.iter().enumerate() {
if needles.contains(&b) {
m |= 1u32 << i;
}
}
m
}
#[test]
fn structural_bitmask_marks_every_needle_position() {
let s = SimdScanner::new(b":-{}\n");
let chunk: [u8; 32] = *b":bcd-fgh:abcdefgh{ijklmnopqrstuv";
let mask = s.structural_bitmask_32(&chunk);
assert_eq!(mask & (1 << 0), 1 << 0);
assert_eq!(mask & (1 << 4), 1 << 4);
assert_eq!(mask & (1 << 8), 1 << 8);
assert_eq!(mask & (1 << 17), 1 << 17);
assert_eq!(mask, (1 << 0) | (1 << 4) | (1 << 8) | (1 << 17));
}
#[test]
fn structural_bitmask_is_zero_for_clean_chunks() {
let s = SimdScanner::new(b":\n");
let chunk = *b"plain text only no delim chars!.";
assert_eq!(s.structural_bitmask_32(&chunk), 0);
}
#[test]
fn structural_bitmask_matches_baseline_across_needle_sets() {
let needle_sets: &[&[u8]] = &[
b":", b":\n", b":,#", b":-[]{}\n", b":-[]{}\n# \t", ];
for &needles in needle_sets {
let s = SimdScanner::new(needles);
for variant in 0..64u8 {
let mut chunk = [0u8; 32];
for (i, slot) in chunk.iter_mut().enumerate() {
*slot = (i as u8).wrapping_add(33).wrapping_add(variant);
while needles.contains(slot) {
*slot = slot.wrapping_add(1);
}
}
for j in (0..32).step_by(3) {
if (variant as usize + j) & 1 == 1 {
chunk[j] = needles[(variant as usize + j) % needles.len()];
}
}
let actual = s.structural_bitmask_32(&chunk);
let expected = baseline_bitmask_32(&chunk, needles);
assert_eq!(
actual, expected,
"needles={needles:?} variant={variant} chunk={chunk:?}"
);
}
}
}
#[test]
fn structural_iter_yields_positions_in_order() {
let s = SimdScanner::new(b":\n");
let positions: Vec<usize> = StructuralIter::new(&s, b"k1: v1\nk2: v2\n").collect();
assert_eq!(positions, vec![2, 6, 9, 13]);
}
#[test]
fn structural_iter_handles_empty_input() {
let s = SimdScanner::new(b":");
let positions: Vec<usize> = StructuralIter::new(&s, b"").collect();
assert!(positions.is_empty());
}
#[test]
fn structural_iter_handles_partial_chunk_tail() {
let s = SimdScanner::new(b":");
let positions: Vec<usize> = StructuralIter::new(&s, b"abc:def:gh").collect();
assert_eq!(positions, vec![3, 7]);
}
#[test]
fn structural_iter_spans_multiple_chunks() {
let mut buf = vec![b'.'; 100];
for &p in &[5usize, 25, 60, 95] {
buf[p] = b':';
}
let s = SimdScanner::new(b":");
let positions: Vec<usize> = StructuralIter::new(&s, &buf).collect();
assert_eq!(positions, vec![5, 25, 60, 95]);
}
#[test]
fn structural_iter_count_matches_scalar_baseline() {
let s = SimdScanner::new(b":\n,#");
let buf: Vec<u8> = (0..2048u32)
.map(|i| if i % 7 == 0 { b':' } else { b'a' })
.collect();
let scalar: Vec<usize> = buf
.iter()
.enumerate()
.filter_map(|(i, &b)| (b == b':').then_some(i))
.collect();
let simd: Vec<usize> = StructuralIter::new(&s, &buf).collect();
assert_eq!(simd, scalar);
}
#[test]
fn parse_decimal_u64_basic() {
assert_eq!(parse_decimal_u64(b"0"), Some(0));
assert_eq!(parse_decimal_u64(b"42"), Some(42));
assert_eq!(parse_decimal_u64(b"12345678"), Some(12_345_678));
assert_eq!(parse_decimal_u64(b"99999999"), Some(99_999_999));
}
#[test]
fn parse_decimal_u64_long_inputs_swar_path() {
assert_eq!(parse_decimal_u64(b"1234567890"), Some(1_234_567_890));
assert_eq!(
parse_decimal_u64(b"9999999999999999"),
Some(9_999_999_999_999_999),
);
assert_eq!(
parse_decimal_u64(b"1234567812345678"),
Some(1_234_567_812_345_678),
);
}
#[test]
fn parse_decimal_u64_max() {
assert_eq!(parse_decimal_u64(b"18446744073709551615"), Some(u64::MAX));
}
#[test]
fn parse_decimal_u64_rejects_non_digits() {
assert_eq!(parse_decimal_u64(b""), None);
assert_eq!(parse_decimal_u64(b"12a4"), None);
assert_eq!(parse_decimal_u64(b" 42"), None);
assert_eq!(parse_decimal_u64(b"42 "), None);
assert_eq!(parse_decimal_u64(b"-42"), None);
assert_eq!(parse_decimal_u64(b"+42"), None);
}
#[test]
fn parse_decimal_u64_overflow_returns_none() {
assert_eq!(parse_decimal_u64(b"18446744073709551616"), None);
assert_eq!(parse_decimal_u64(b"99999999999999999999999"), None,);
}
#[test]
fn parse_decimal_u64_matches_stdlib_baseline() {
for n in [
0u64,
1,
9,
10,
99,
100,
999,
1000,
12345,
1234567,
12345678,
123456789,
9876543210,
1234567890123456,
9_223_372_036_854_775_807, 10_000_000_000_000_000_000,
u64::MAX - 1,
u64::MAX,
] {
let s = n.to_string();
assert_eq!(parse_decimal_u64(s.as_bytes()), Some(n), "n={n}");
}
}
#[test]
fn parse_decimal_i64_handles_signs() {
assert_eq!(parse_decimal_i64(b"42"), Some(42));
assert_eq!(parse_decimal_i64(b"+42"), Some(42));
assert_eq!(parse_decimal_i64(b"-42"), Some(-42));
assert_eq!(parse_decimal_i64(b"0"), Some(0));
assert_eq!(parse_decimal_i64(b"-0"), Some(0));
}
#[test]
fn parse_decimal_i64_handles_min_max() {
assert_eq!(parse_decimal_i64(b"9223372036854775807"), Some(i64::MAX));
assert_eq!(parse_decimal_i64(b"-9223372036854775808"), Some(i64::MIN));
}
#[test]
fn parse_decimal_i64_rejects_overflow() {
assert_eq!(parse_decimal_i64(b"9223372036854775808"), None);
assert_eq!(parse_decimal_i64(b"-9223372036854775809"), None);
}
#[test]
fn parse_decimal_i64_matches_stdlib_across_full_range() {
for n in [
0i64,
1,
-1,
10,
-10,
100,
-100,
i32::MIN as i64,
i32::MAX as i64,
i64::MIN,
i64::MAX,
i64::MAX - 1,
i64::MIN + 1,
-123_456_789,
987_654_321,
] {
let s = n.to_string();
assert_eq!(parse_decimal_i64(s.as_bytes()), Some(n), "n={n}");
}
}
}