use crate::{estimate_from_whitespace_splits, TokenCounter};
const HIGH: u64 = 0x8080_8080_8080_8080;
const SEVENS: u64 = 0x7F7F_7F7F_7F7F_7F7F;
const ADD_5F: u64 = 0x5F5F_5F5F_5F5F_5F5F;
#[inline(always)]
fn nonwhitespace_mask(word: u64) -> u64 {
(word | ((word & SEVENS) + ADD_5F)) & HIGH
}
#[derive(Debug, Clone, Default)]
pub struct SimdWhitespaceTokenizer {
_priv: (), }
impl SimdWhitespaceTokenizer {
pub fn new() -> Self {
Self { _priv: () }
}
pub fn count_splits(&self, bytes: &[u8]) -> usize {
count_splits_swar(bytes)
}
}
impl TokenCounter for SimdWhitespaceTokenizer {
fn count(&self, text: &str) -> usize {
let bytes = text.as_bytes();
let splits = count_splits_swar(bytes);
estimate_from_whitespace_splits(splits, bytes.len())
}
fn backend_name(&self) -> &'static str {
"simd-whitespace"
}
}
fn count_splits_swar(data: &[u8]) -> usize {
let mut count = 0usize;
let mut prev_ws = true;
let mut chunks = data.chunks_exact(8);
for chunk in &mut chunks {
#[allow(clippy::expect_used)]
let word = u64::from_le_bytes(chunk.try_into().expect("chunks_exact yields 8 bytes"));
let nonws = nonwhitespace_mask(word); let ws = !nonws & HIGH;
let starts = nonws & ((ws << 8) | if prev_ws { 0x80 } else { 0 });
count += (starts >> 7).count_ones() as usize;
prev_ws = ws & (0x80 << 56) != 0;
}
for &b in chunks.remainder() {
let is_ws = b <= 0x20;
if !is_ws && prev_ws {
count += 1;
}
prev_ws = is_ws;
}
count
}
pub fn scalar_count_splits(data: &[u8]) -> usize {
let mut in_word = false;
let mut count = 0;
for &b in data {
let is_ws = b <= 0x20;
if is_ws {
if in_word {
count += 1;
in_word = false;
}
} else {
in_word = true;
}
}
if in_word {
count += 1;
}
count
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_scalar_empty() {
assert_eq!(scalar_count_splits(b""), 0);
}
#[test]
fn test_scalar_single_word() {
assert_eq!(scalar_count_splits(b"hello"), 1);
}
#[test]
fn test_scalar_two_words() {
assert_eq!(scalar_count_splits(b"hello world"), 2);
}
#[test]
fn test_scalar_whitespace_only() {
assert_eq!(scalar_count_splits(b" \t\n"), 0);
}
#[test]
fn test_scalar_mixed_whitespace() {
assert_eq!(scalar_count_splits(b"a b\tc\nd"), 4);
}
#[test]
fn test_scalar_leading_trailing_ws() {
assert_eq!(scalar_count_splits(b" hello world "), 2);
}
#[test]
fn test_scalar_c0_controls() {
assert_eq!(scalar_count_splits(b"a\x00b\x01c"), 3);
}
#[test]
fn test_simd_tokenizer_empty() {
let t = SimdWhitespaceTokenizer::new();
assert_eq!(t.count(""), 0);
}
#[test]
fn test_simd_tokenizer_single() {
let t = SimdWhitespaceTokenizer::new();
assert!(t.count("hello") >= 1);
}
#[test]
fn test_simd_tokenizer_sentence() {
let t = SimdWhitespaceTokenizer::new();
let c = t.count("The quick brown fox jumps over the lazy dog.");
assert!(c >= 5);
}
#[test]
fn test_simd_tokenizer_code() {
let t = SimdWhitespaceTokenizer::new();
let code = "fn main() { let x = 42; }";
let c = t.count(code);
assert!(c >= 4);
}
#[test]
fn test_simd_tokenizer_unicode() {
let t = SimdWhitespaceTokenizer::new();
let c = t.count("こんにちは 世界 hello");
assert!(c >= 2);
}
#[test]
fn test_simd_matches_scalar() {
let t = SimdWhitespaceTokenizer::new();
let cases: &[&str] = &[
"",
"a",
"hello world",
" leading and trailing ",
"tabs\there\ttoo",
"new\nlines\nhere",
];
let long = "The quick brown fox jumps over the lazy dog. ".repeat(10);
for case in cases.iter().copied().chain(core::iter::once(long.as_str())) {
assert_eq!(
t.count_splits(case.as_bytes()),
scalar_count_splits(case.as_bytes()),
"mismatch for input of {} bytes",
case.len()
);
}
}
#[test]
fn test_17_byte_alignment() {
let data = b"12345678901234567";
let t = SimdWhitespaceTokenizer::new();
assert_eq!(t.count_splits(data), scalar_count_splits(data));
}
#[test]
fn test_33_byte_alignment() {
let data = b"123456789012345678901234567890123";
let t = SimdWhitespaceTokenizer::new();
assert_eq!(t.count_splits(data), scalar_count_splits(data));
}
#[test]
fn test_exact_16_bytes() {
let data = b"1234567890123456";
let t = SimdWhitespaceTokenizer::new();
assert_eq!(t.count_splits(data), scalar_count_splits(data));
}
#[test]
fn test_exact_32_bytes() {
let data = b"12345678901234567890123456789012";
let t = SimdWhitespaceTokenizer::new();
assert_eq!(t.count_splits(data), scalar_count_splits(data));
}
#[test]
fn test_empty_string() {
let t = SimdWhitespaceTokenizer::new();
assert_eq!(t.count(""), 0);
}
#[test]
fn test_single_word() {
let t = SimdWhitespaceTokenizer::new();
let c = t.count("hello");
assert!(c >= 1, "single word should yield >= 1 token, got {c}");
}
#[test]
fn test_simple_sentence() {
let t = SimdWhitespaceTokenizer::new();
let c = t.count("The quick brown fox jumps over the lazy dog.");
assert!(c >= 5, "sentence should yield >= 5 tokens, got {c}");
}
#[test]
fn test_long_text() {
let t = SimdWhitespaceTokenizer::new();
let text = "The quick brown fox jumps over the lazy dog. ".repeat(100);
let c = t.count(&text);
assert!(c >= 100, "long text should yield >= 100 tokens, got {c}");
}
#[test]
fn test_unicode_text() {
let t = SimdWhitespaceTokenizer::new();
let c = t.count("こんにちは世界 Hello 世界 this is unicode 🌍");
assert!(c >= 3, "unicode text should yield >= 3 tokens, got {c}");
}
#[test]
fn test_code_with_symbols() {
let t = SimdWhitespaceTokenizer::new();
let code = r#"fn main() {
let x = 42;
println!("Hello, world!");
if x > 0 {
println!("positive");
}
}"#;
let c = t.count(code);
assert!(c >= 8, "code should yield >= 8 tokens, got {c}");
}
#[test]
fn test_whitespace_only() {
let t = SimdWhitespaceTokenizer::new();
assert_eq!(t.count(" \t\n\r "), 0);
}
#[test]
fn test_single_character() {
let t = SimdWhitespaceTokenizer::new();
assert_eq!(t.count("a"), 1);
}
#[test]
fn test_repeated_spaces() {
let t = SimdWhitespaceTokenizer::new();
let c = t.count("hello world");
assert_eq!(t.count("hello world"), c);
}
#[test]
fn test_newlines_and_tabs() {
let t = SimdWhitespaceTokenizer::new();
let c1 = t.count("hello\nworld");
let c2 = t.count("hello\tworld");
let c3 = t.count("hello world");
assert_eq!(c1, c2);
assert_eq!(c2, c3);
}
#[test]
fn test_backend_name() {
let t = SimdWhitespaceTokenizer::new();
assert_eq!(t.backend_name(), "simd-whitespace");
}
#[test]
fn test_estimate_from_splits_empty() {
assert_eq!(estimate_from_whitespace_splits(0, 0), 0);
}
#[test]
fn test_estimate_from_splits_single() {
let est = estimate_from_whitespace_splits(1, 5);
assert!(est >= 1);
}
#[test]
fn test_large_input_no_panic() {
let t = SimdWhitespaceTokenizer::new();
let text = "word ".repeat(100_000);
let _ = t.count(&text);
}
#[test]
fn test_emoji_heavy() {
let t = SimdWhitespaceTokenizer::new();
let c = t.count("😀😂🤣😃😄😁😆😅🤪😊😎");
assert!(c >= 1);
}
}