pub(crate) const MODE_STANDARD: u8 = 0;
pub(crate) const MODE_SCRIPT_SAFE: u8 = 1;
pub(crate) const MODE_ASCII_ONLY: u8 = 2;
pub(crate) const MODE_HTML_SAFE: u8 = 3;
pub(crate) const MODE_HTML_ENTITIES: u8 = 4;
pub(crate) const MODE_JS_SEPARATORS: u8 = 5;
pub(crate) const JS_SEP_LEAD: u8 = 0xE2;
pub(crate) const JS_SEP_CONT: u8 = 0x80;
pub(crate) const JS_SEP_LAST_2028: u8 = 0xA8;
pub(crate) const JS_SEP_LAST_2029: u8 = 0xA9;
pub(crate) const CLASS_STANDARD: u8 = 1 << 0;
pub(crate) const CLASS_SCRIPT_SAFE_EXTRA: u8 = 1 << 1;
pub(crate) const CLASS_ASCII_ONLY_EXTRA: u8 = 1 << 2;
pub(crate) const CLASS_HTML_SAFE_EXTRA: u8 = 1 << 3;
pub(crate) const CLASS_HTML_ENTITIES_EXTRA: u8 = 1 << 4;
pub(crate) const CLASS_JS_SEPARATORS_EXTRA: u8 = 1 << 5;
pub(crate) const fn class_bits<const MODE: u8>() -> u8 {
match MODE {
MODE_SCRIPT_SAFE => CLASS_STANDARD | CLASS_SCRIPT_SAFE_EXTRA,
MODE_ASCII_ONLY => CLASS_STANDARD | CLASS_ASCII_ONLY_EXTRA,
MODE_HTML_SAFE => CLASS_STANDARD | CLASS_HTML_SAFE_EXTRA,
MODE_HTML_ENTITIES => CLASS_STANDARD | CLASS_HTML_ENTITIES_EXTRA,
MODE_JS_SEPARATORS => CLASS_STANDARD | CLASS_JS_SEPARATORS_EXTRA,
_ => CLASS_STANDARD,
}
}
pub(crate) static ESCAPE_CLASS: [u8; 256] = {
let mut t = [0u8; 256];
let mut i = 0;
while i < 256 {
let b = i as u8;
if b == b'"' || b == b'\\' || b < 0x20 {
t[i] |= CLASS_STANDARD;
}
if b == b'/' || b == JS_SEP_LEAD {
t[i] |= CLASS_SCRIPT_SAFE_EXTRA;
}
if b >= 0x80 {
t[i] |= CLASS_ASCII_ONLY_EXTRA;
}
if b == b'<' || b == b'>' || b == b'&' || b == JS_SEP_LEAD {
t[i] |= CLASS_HTML_SAFE_EXTRA;
}
if b == b'<' || b == b'>' || b == b'&' {
t[i] |= CLASS_HTML_ENTITIES_EXTRA;
}
if b == JS_SEP_LEAD {
t[i] |= CLASS_JS_SEPARATORS_EXTRA;
}
i += 1;
}
t
};
#[inline(always)]
pub(crate) unsafe fn copy_small(src: *const u8, dst: *mut u8, n: usize) {
debug_assert!(n <= 32);
macro_rules! word_pair {
($t:ty) => {{
const SIZE: usize = size_of::<$t>();
dst.cast::<$t>()
.write_unaligned(src.cast::<$t>().read_unaligned());
dst.add(n - SIZE)
.cast::<$t>()
.write_unaligned(src.add(n - SIZE).cast::<$t>().read_unaligned());
}};
}
unsafe {
if n >= 16 {
word_pair!(u128);
} else if n >= 8 {
word_pair!(u64);
} else if n >= 4 {
word_pair!(u32);
} else if n >= 2 {
word_pair!(u16);
} else if n == 1 {
dst.write(src.read());
}
}
}
#[inline(always)]
pub(crate) fn padded_tail<const W: usize>(tail: &[u8]) -> [u8; W] {
debug_assert!(tail.len() < W);
let mut buf = [0u8; W];
unsafe { copy_small(tail.as_ptr(), buf.as_mut_ptr(), tail.len()) };
buf
}
#[cfg(any(test, not(any(target_arch = "aarch64", target_arch = "x86_64"))))]
#[inline(always)]
pub(crate) fn swar_escape_mask<const MODE: u8>(w: u64) -> u64 {
const LO: u64 = 0x0101_0101_0101_0101;
const HI: u64 = 0x8080_8080_8080_8080;
#[inline(always)]
fn eq(w: u64, b: u8) -> u64 {
let v = w ^ (LO * u64::from(b));
v.wrapping_sub(LO) & !v & HI
}
let mut m = eq(w, b'"') | eq(w, b'\\') | (w.wrapping_sub(LO * 0x20) & !w & HI);
if MODE == MODE_SCRIPT_SAFE {
m |= eq(w, b'/') | eq(w, JS_SEP_LEAD);
}
if MODE == MODE_ASCII_ONLY {
m |= w & HI;
}
if MODE == MODE_HTML_SAFE {
m |= eq(w, b'<') | eq(w, b'>') | eq(w, b'&') | eq(w, JS_SEP_LEAD);
}
if MODE == MODE_HTML_ENTITIES {
m |= eq(w, b'<') | eq(w, b'>') | eq(w, b'&');
}
if MODE == MODE_JS_SEPARATORS {
m |= eq(w, JS_SEP_LEAD);
}
m
}
#[cfg(target_arch = "aarch64")]
pub(crate) mod neon {
use super::{
JS_SEP_LEAD, MODE_ASCII_ONLY, MODE_HTML_ENTITIES, MODE_HTML_SAFE, MODE_JS_SEPARATORS,
MODE_SCRIPT_SAFE,
};
use std::arch::aarch64::{
uint8x16_t, vceqq_u8, vcgeq_u8, vcltq_u8, vdupq_n_u8, vget_lane_u64, vld1q_u8, vorrq_u8,
vreinterpret_u64_u8, vreinterpretq_u16_u8, vshrn_n_u16, vst1q_u8,
};
#[inline(always)]
pub(crate) unsafe fn hit_vec<const MODE: u8>(v: uint8x16_t) -> uint8x16_t {
unsafe {
let mut hit = vorrq_u8(
vorrq_u8(
vceqq_u8(v, vdupq_n_u8(b'"')),
vceqq_u8(v, vdupq_n_u8(b'\\')),
),
vcltq_u8(v, vdupq_n_u8(0x20)),
);
if MODE == MODE_SCRIPT_SAFE {
hit = vorrq_u8(
hit,
vorrq_u8(
vceqq_u8(v, vdupq_n_u8(b'/')),
vceqq_u8(v, vdupq_n_u8(JS_SEP_LEAD)),
),
);
}
if MODE == MODE_ASCII_ONLY {
hit = vorrq_u8(hit, vcgeq_u8(v, vdupq_n_u8(0x80)));
}
if MODE == MODE_HTML_SAFE {
hit = vorrq_u8(
hit,
vorrq_u8(
vorrq_u8(vceqq_u8(v, vdupq_n_u8(b'<')), vceqq_u8(v, vdupq_n_u8(b'>'))),
vorrq_u8(
vceqq_u8(v, vdupq_n_u8(b'&')),
vceqq_u8(v, vdupq_n_u8(JS_SEP_LEAD)),
),
),
);
}
if MODE == MODE_HTML_ENTITIES {
hit = vorrq_u8(
hit,
vorrq_u8(
vorrq_u8(vceqq_u8(v, vdupq_n_u8(b'<')), vceqq_u8(v, vdupq_n_u8(b'>'))),
vceqq_u8(v, vdupq_n_u8(b'&')),
),
);
}
if MODE == MODE_JS_SEPARATORS {
hit = vorrq_u8(hit, vceqq_u8(v, vdupq_n_u8(JS_SEP_LEAD)));
}
hit
}
}
#[inline(always)]
pub(crate) unsafe fn nib_mask(hit: uint8x16_t) -> u64 {
unsafe {
vget_lane_u64::<0>(vreinterpret_u64_u8(vshrn_n_u16::<4>(vreinterpretq_u16_u8(
hit,
))))
}
}
#[inline(always)]
pub(crate) unsafe fn hit_mask<const MODE: u8>(v: uint8x16_t) -> u64 {
unsafe { nib_mask(hit_vec::<MODE>(v)) }
}
#[inline(always)]
pub(crate) unsafe fn copy_scan<const MODE: u8>(src: *const u8, dst: *mut u8) -> u64 {
unsafe {
let v = vld1q_u8(src);
vst1q_u8(dst, v);
hit_mask::<MODE>(v)
}
}
#[inline(always)]
pub(crate) fn tail_find<const MODE: u8>(input: &[u8], from: usize) -> Option<usize> {
let back = input.len().checked_sub(16)?;
debug_assert!(from > back && from < input.len());
let mask = unsafe { hit_mask::<MODE>(vld1q_u8(input.as_ptr().add(back))) }
& (u64::MAX << (4 * (from - back)));
if mask == 0 {
None
} else {
Some(back + (mask.trailing_zeros() as usize) / 4)
}
}
}
#[cfg(target_arch = "x86_64")]
pub(crate) mod x86 {
use super::{
JS_SEP_LEAD, MODE_ASCII_ONLY, MODE_HTML_ENTITIES, MODE_HTML_SAFE, MODE_JS_SEPARATORS,
MODE_SCRIPT_SAFE,
};
use std::arch::x86_64::{
__m128i, __m256i, __m512i, _mm_cmpeq_epi8, _mm_loadu_si128, _mm_min_epu8,
_mm_movemask_epi8, _mm_or_si128, _mm_set1_epi8, _mm_storeu_si128, _mm256_cmpeq_epi8,
_mm256_loadu_si256, _mm256_min_epu8, _mm256_movemask_epi8, _mm256_or_si256,
_mm256_set1_epi8, _mm256_storeu_si256, _mm512_cmpeq_epi8_mask, _mm512_cmplt_epu8_mask,
_mm512_loadu_si512, _mm512_movepi8_mask, _mm512_set1_epi8, _mm512_storeu_si512,
};
#[inline(always)]
pub(crate) unsafe fn sse2_hit_mask<const MODE: u8>(v: __m128i) -> u32 {
unsafe {
let is_ctrl = _mm_cmpeq_epi8(_mm_min_epu8(v, _mm_set1_epi8(0x1F)), v);
let mut hit = _mm_or_si128(
_mm_or_si128(
_mm_cmpeq_epi8(v, _mm_set1_epi8(b'"' as i8)),
_mm_cmpeq_epi8(v, _mm_set1_epi8(b'\\' as i8)),
),
is_ctrl,
);
if MODE == MODE_SCRIPT_SAFE {
hit = _mm_or_si128(
hit,
_mm_or_si128(
_mm_cmpeq_epi8(v, _mm_set1_epi8(b'/' as i8)),
_mm_cmpeq_epi8(v, _mm_set1_epi8(JS_SEP_LEAD as i8)),
),
);
}
if MODE == MODE_HTML_SAFE {
hit = _mm_or_si128(
hit,
_mm_or_si128(
_mm_or_si128(
_mm_cmpeq_epi8(v, _mm_set1_epi8(b'<' as i8)),
_mm_cmpeq_epi8(v, _mm_set1_epi8(b'>' as i8)),
),
_mm_or_si128(
_mm_cmpeq_epi8(v, _mm_set1_epi8(b'&' as i8)),
_mm_cmpeq_epi8(v, _mm_set1_epi8(JS_SEP_LEAD as i8)),
),
),
);
}
if MODE == MODE_HTML_ENTITIES {
hit = _mm_or_si128(
hit,
_mm_or_si128(
_mm_or_si128(
_mm_cmpeq_epi8(v, _mm_set1_epi8(b'<' as i8)),
_mm_cmpeq_epi8(v, _mm_set1_epi8(b'>' as i8)),
),
_mm_cmpeq_epi8(v, _mm_set1_epi8(b'&' as i8)),
),
);
}
if MODE == MODE_JS_SEPARATORS {
hit = _mm_or_si128(hit, _mm_cmpeq_epi8(v, _mm_set1_epi8(JS_SEP_LEAD as i8)));
}
let mut mask = _mm_movemask_epi8(hit) as u32;
if MODE == MODE_ASCII_ONLY {
mask |= _mm_movemask_epi8(v) as u32;
}
mask
}
}
#[target_feature(enable = "avx2")]
#[inline]
pub(crate) unsafe fn avx2_hit_mask<const MODE: u8>(v: __m256i) -> u32 {
let is_ctrl = _mm256_cmpeq_epi8(_mm256_min_epu8(v, _mm256_set1_epi8(0x1F)), v);
let mut hit = _mm256_or_si256(
_mm256_or_si256(
_mm256_cmpeq_epi8(v, _mm256_set1_epi8(b'"' as i8)),
_mm256_cmpeq_epi8(v, _mm256_set1_epi8(b'\\' as i8)),
),
is_ctrl,
);
if MODE == MODE_SCRIPT_SAFE {
hit = _mm256_or_si256(
hit,
_mm256_or_si256(
_mm256_cmpeq_epi8(v, _mm256_set1_epi8(b'/' as i8)),
_mm256_cmpeq_epi8(v, _mm256_set1_epi8(JS_SEP_LEAD as i8)),
),
);
}
if MODE == MODE_HTML_SAFE {
hit = _mm256_or_si256(
hit,
_mm256_or_si256(
_mm256_or_si256(
_mm256_cmpeq_epi8(v, _mm256_set1_epi8(b'<' as i8)),
_mm256_cmpeq_epi8(v, _mm256_set1_epi8(b'>' as i8)),
),
_mm256_or_si256(
_mm256_cmpeq_epi8(v, _mm256_set1_epi8(b'&' as i8)),
_mm256_cmpeq_epi8(v, _mm256_set1_epi8(JS_SEP_LEAD as i8)),
),
),
);
}
if MODE == MODE_HTML_ENTITIES {
hit = _mm256_or_si256(
hit,
_mm256_or_si256(
_mm256_or_si256(
_mm256_cmpeq_epi8(v, _mm256_set1_epi8(b'<' as i8)),
_mm256_cmpeq_epi8(v, _mm256_set1_epi8(b'>' as i8)),
),
_mm256_cmpeq_epi8(v, _mm256_set1_epi8(b'&' as i8)),
),
);
}
if MODE == MODE_JS_SEPARATORS {
hit = _mm256_or_si256(
hit,
_mm256_cmpeq_epi8(v, _mm256_set1_epi8(JS_SEP_LEAD as i8)),
);
}
let mut mask = _mm256_movemask_epi8(hit) as u32;
if MODE == MODE_ASCII_ONLY {
mask |= _mm256_movemask_epi8(v) as u32;
}
mask
}
#[inline(always)]
pub(crate) unsafe fn sse2_copy_scan<const MODE: u8>(src: *const u8, dst: *mut u8) -> u64 {
unsafe {
let v = _mm_loadu_si128(src.cast());
_mm_storeu_si128(dst.cast(), v);
u64::from(sse2_hit_mask::<MODE>(v))
}
}
#[target_feature(enable = "avx2")]
#[inline]
pub(crate) unsafe fn avx2_copy_scan<const MODE: u8>(src: *const u8, dst: *mut u8) -> u64 {
unsafe {
let v = _mm256_loadu_si256(src.cast());
_mm256_storeu_si256(dst.cast(), v);
u64::from(avx2_hit_mask::<MODE>(v))
}
}
#[target_feature(enable = "avx512bw")]
#[inline]
#[allow(dead_code)]
pub(crate) unsafe fn avx512_hit_mask<const MODE: u8>(v: __m512i) -> u64 {
let mut mask = _mm512_cmpeq_epi8_mask(v, _mm512_set1_epi8(b'"' as i8))
| _mm512_cmpeq_epi8_mask(v, _mm512_set1_epi8(b'\\' as i8))
| _mm512_cmplt_epu8_mask(v, _mm512_set1_epi8(0x20));
if MODE == MODE_SCRIPT_SAFE {
mask |= _mm512_cmpeq_epi8_mask(v, _mm512_set1_epi8(b'/' as i8))
| _mm512_cmpeq_epi8_mask(v, _mm512_set1_epi8(JS_SEP_LEAD as i8));
}
if MODE == MODE_HTML_SAFE {
mask |= _mm512_cmpeq_epi8_mask(v, _mm512_set1_epi8(b'<' as i8))
| _mm512_cmpeq_epi8_mask(v, _mm512_set1_epi8(b'>' as i8))
| _mm512_cmpeq_epi8_mask(v, _mm512_set1_epi8(b'&' as i8))
| _mm512_cmpeq_epi8_mask(v, _mm512_set1_epi8(JS_SEP_LEAD as i8));
}
if MODE == MODE_HTML_ENTITIES {
mask |= _mm512_cmpeq_epi8_mask(v, _mm512_set1_epi8(b'<' as i8))
| _mm512_cmpeq_epi8_mask(v, _mm512_set1_epi8(b'>' as i8))
| _mm512_cmpeq_epi8_mask(v, _mm512_set1_epi8(b'&' as i8));
}
if MODE == MODE_JS_SEPARATORS {
mask |= _mm512_cmpeq_epi8_mask(v, _mm512_set1_epi8(JS_SEP_LEAD as i8));
}
if MODE == MODE_ASCII_ONLY {
mask |= _mm512_movepi8_mask(v) as u64;
}
mask
}
#[target_feature(enable = "avx512bw")]
#[inline]
#[allow(dead_code)]
pub(crate) unsafe fn avx512_copy_scan<const MODE: u8>(src: *const u8, dst: *mut u8) -> u64 {
unsafe {
let v = _mm512_loadu_si512(src.cast());
_mm512_storeu_si512(dst.cast(), v);
avx512_hit_mask::<MODE>(v)
}
}
#[inline(always)]
pub(crate) fn sse2_tail_find<const MODE: u8>(input: &[u8], from: usize) -> Option<usize> {
let back = input.len().checked_sub(16)?;
debug_assert!(from > back && from < input.len());
let mask =
unsafe { sse2_hit_mask::<MODE>(_mm_loadu_si128(input.as_ptr().add(back).cast())) }
& (u32::MAX << (from - back));
if mask == 0 {
None
} else {
Some(back + mask.trailing_zeros() as usize)
}
}
#[target_feature(enable = "avx2")]
#[inline]
pub(crate) fn avx2_tail_find<const MODE: u8>(input: &[u8], from: usize) -> Option<usize> {
let back = input.len().checked_sub(32)?;
debug_assert!(from > back && from < input.len());
let mask =
unsafe { avx2_hit_mask::<MODE>(_mm256_loadu_si256(input.as_ptr().add(back).cast())) }
& (u32::MAX << (from - back));
if mask == 0 {
None
} else {
Some(back + mask.trailing_zeros() as usize)
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn swar_agrees_with_class_table() {
fn check<const MODE: u8>() {
for b in 0..=255u8 {
let word = u64::from_le_bytes([b, b'a', b, 0x00, b'z', b, b'a', b]);
let mask = swar_escape_mask::<MODE>(word);
for (lane, &byte) in [b, b'a', b, 0x00, b'z', b, b'a', b].iter().enumerate() {
let expected = ESCAPE_CLASS[byte as usize] & class_bits::<MODE>() != 0;
let got = mask & (0x80 << (8 * lane)) != 0;
assert_eq!(got, expected, "mode {MODE} byte {byte:#x} lane {lane}");
}
}
}
check::<MODE_STANDARD>();
check::<MODE_SCRIPT_SAFE>();
check::<MODE_ASCII_ONLY>();
check::<MODE_HTML_SAFE>();
check::<MODE_HTML_ENTITIES>();
check::<MODE_JS_SEPARATORS>();
}
}