memspan 0.1.0

SIMD-accelerated byte-class scanning for lexers and parsers. Backends: AVX-512, AVX2, SSE4.1, NEON, WASM SIMD128. no_std compatible.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138

use memspan::skip;

#[test]
fn skip_binary_basic() {
  assert_eq!(skip::skip_binary(b""), 0);
  assert_eq!(skip::skip_binary(b"0"), 1);
  assert_eq!(skip::skip_binary(b"1"), 1);
  assert_eq!(skip::skip_binary(b"01010101"), 8);
  assert_eq!(skip::skip_binary(b"012"), 2); // '2' not binary
  assert_eq!(skip::skip_binary(b"a01"), 0);
}

#[test]
fn skip_binary_chunk_boundaries_exhaustive() {
  for len in 1usize..=80 {
    // miss at each position
    for miss_pos in 0..len {
      let mut input = vec![b'0'; len];
      input[miss_pos] = b'2';
      assert_eq!(
        skip::skip_binary(&input),
        miss_pos,
        "len={len}, miss_pos={miss_pos}"
      );
    }
    // all-match
    let input = vec![b'1'; len];
    assert_eq!(skip::skip_binary(&input), len, "len={len}, all-match");
  }
}

#[test]
fn skip_octal_basic() {
  assert_eq!(skip::skip_octal_digits(b""), 0);
  assert_eq!(skip::skip_octal_digits(b"01234567"), 8);
  assert_eq!(skip::skip_octal_digits(b"0128"), 3); // '8' at idx 3 not octal
  assert_eq!(skip::skip_octal_digits(b"017a"), 3); // 'a' at idx 3
  assert_eq!(skip::skip_octal_digits(b"79a"), 1); // '9' at idx 1 not octal
}

#[test]
fn skip_octal_chunk_boundaries_exhaustive() {
  for len in 1usize..=80 {
    for miss_pos in 0..len {
      let mut input = vec![b'7'; len];
      input[miss_pos] = b'8';
      assert_eq!(
        skip::skip_octal_digits(&input),
        miss_pos,
        "len={len}, miss_pos={miss_pos}"
      );
    }
    let input = vec![b'5'; len];
    assert_eq!(skip::skip_octal_digits(&input), len, "len={len}, all-match");
  }
}

#[test]
fn skip_digits_basic() {
  assert_eq!(skip::skip_digits(b""), 0);
  assert_eq!(skip::skip_digits(b"0123456789"), 10);
  assert_eq!(skip::skip_digits(b"123abc"), 3);
  assert_eq!(skip::skip_digits(b"a123"), 0);
}

#[test]
fn skip_digits_chunk_boundaries_exhaustive() {
  for len in 1usize..=80 {
    for miss_pos in 0..len {
      let mut input = vec![b'5'; len];
      input[miss_pos] = b'a';
      assert_eq!(
        skip::skip_digits(&input),
        miss_pos,
        "len={len}, miss_pos={miss_pos}"
      );
    }
    let input = vec![b'9'; len];
    assert_eq!(skip::skip_digits(&input), len, "len={len}, all-match");
  }
}

#[test]
fn skip_hex_digits_basic() {
  assert_eq!(skip::skip_hex_digits(b""), 0);
  assert_eq!(skip::skip_hex_digits(b"deadBEEF"), 8);
  assert_eq!(skip::skip_hex_digits(b"0123456789abcdefABCDEF"), 22);
  assert_eq!(skip::skip_hex_digits(b"ffg"), 2); // 'g' is not hex
  assert_eq!(skip::skip_hex_digits(b"FFG"), 2); // 'G' is not hex
  assert_eq!(skip::skip_hex_digits(b"123:"), 3); // ':' (0x3a) not hex
  assert_eq!(skip::skip_hex_digits(b"@"), 0); // '@' (0x40 → 0x60 after |0x20) not hex
  assert_eq!(skip::skip_hex_digits(b"`"), 0); // '`' (0x60) not hex
  assert_eq!(skip::skip_hex_digits(b"/"), 0); // '/' (0x2f) just below '0'
}

/// The case-fold trick (`byte | 0x20`) is what makes `[A-F]` map to `[a-f]`.
/// Make sure it doesn't accidentally accept the shifted forms of nearby
/// punctuation.
#[test]
fn skip_hex_digits_case_fold_boundary_chars() {
  // For every byte, decide if it should be hex-digit-able and check.
  for b in 0u8..=255u8 {
    let expected_hex = b.is_ascii_hexdigit();
    let input = [b];
    let result = skip::skip_hex_digits(&input);
    let want = if expected_hex { 1 } else { 0 };
    assert_eq!(
      result, want,
      "byte 0x{b:02x} ({:?}) — expected hex={expected_hex}",
      b as char
    );
  }
}

#[test]
fn skip_hex_digits_chunk_boundaries_exhaustive() {
  for len in 1usize..=80 {
    for miss_pos in 0..len {
      let mut input = vec![b'a'; len];
      input[miss_pos] = b'g';
      assert_eq!(
        skip::skip_hex_digits(&input),
        miss_pos,
        "len={len}, miss_pos={miss_pos}"
      );
    }
    let input = vec![b'F'; len];
    assert_eq!(skip::skip_hex_digits(&input), len, "len={len}, all-match");
  }
}

/// Mixed case at every position to exercise the OR-with-0x20 inside the SIMD
/// hex-digit mask.
#[test]
fn skip_hex_digits_mixed_case_long() {
  let input = b"0123456789aBcDeFAbCdEf0123456789ABCDEFabcdef0123456789FAFaFAFa";
  assert_eq!(skip::skip_hex_digits(input), input.len());
}