mod accept;
pub(crate) use accept::{AcceptCode, derive_accept};
use crate::oniguruma::{
ONIG_OPTION_FIND_NOT_EMPTY, ONIG_OPTION_NOTBOL, ONIG_OPTION_NOTEOL, ONIGENC_CTYPE_WORD,
};
use crate::regcomp::literal_alt_trie_index;
use crate::regenc::OnigEncoding;
use crate::regint::*;
use crate::regparse_types::*;
use regex_automata::hybrid::LazyStateID;
use regex_automata::hybrid::dfa as hybrid;
use regex_automata::hybrid::regex as hybrid_regex;
use regex_automata::nfa::thompson;
use regex_automata::util::pool::Pool;
use regex_automata::{Anchored, Input, MatchKind, PatternID};
use regex_syntax::hir::{
Class, ClassBytes, ClassBytesRange, ClassUnicode, ClassUnicodeRange, Hir, HirKind, Look,
Repetition,
};
use regex_syntax::utf8::Utf8Sequences;
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
use std::sync::{Arc, OnceLock};
#[cfg(feature = "scanner-group-nfa-prototype")]
pub(crate) mod group_archive;
#[cfg(feature = "precompiled-eager")]
mod portable;
const MIB: usize = 1 << 20;
const SEARCH_CACHE_CAPACITY: usize = 2 * MIB;
const SEARCH_MIN_BYTES_PER_STATE: usize = 10;
const OVERLAPPING_CACHE_CAPACITY: usize = MIB;
const NFA_SIZE_LIMIT: usize = 16 * MIB;
const MIN_CACHE_CLEARS: usize = 3;
const MIN_BYTES_PER_STATE: usize = 20;
const MAX_COUNTED_REPEAT: u32 = 64;
pub(crate) const CANDIDATE_SCAN_BYTES: usize = 256;
pub(crate) const LONG_SEEK_BYTES: usize = 16;
pub(crate) const LONG_WALK_BYTES: usize = 64;
pub(crate) const MAX_NFA_STATES: usize = 65_536;
const MAX_SINGLE_DENSE_ESTIMATED_NFA_STATES: usize = 512;
const MAX_SINGLE_BITSET_NFA_STATES: usize = u64::BITS as usize;
const SINGLE_DENSE_DFA_SIZE_LIMIT: usize = 64 * 1024;
const SINGLE_DENSE_DETERMINIZE_SIZE_LIMIT: usize = 32 * 1024;
pub(crate) const TINY_SET_ENTRIES: usize = 12;
pub(crate) const TINY_SET_NFA_STATES: usize = 128;
#[cfg(test)]
thread_local! {
pub(crate) static EARLIEST_CALLS: std::cell::Cell<u64> = const { std::cell::Cell::new(0) };
pub(crate) static SCAN_STEPS: std::cell::Cell<u64> = const { std::cell::Cell::new(0) };
pub(crate) static META_BYTES: std::cell::Cell<u64> = const { std::cell::Cell::new(0) };
pub(crate) static OVERLAPPING_CACHE_BYTES: std::cell::Cell<Option<usize>> =
const { std::cell::Cell::new(None) };
}
fn overlapping_config() -> hybrid::Config {
let config = hybrid::Config::new()
.match_kind(MatchKind::All)
.skip_cache_capacity_check(true);
#[cfg(test)]
if let Some(bytes) = OVERLAPPING_CACHE_BYTES.with(|bytes| bytes.get()) {
return config
.cache_capacity(bytes)
.minimum_cache_clear_count(None)
.minimum_bytes_per_state(None);
}
config
.cache_capacity(OVERLAPPING_CACHE_CAPACITY)
.minimum_cache_clear_count(Some(MIN_CACHE_CLEARS))
.minimum_bytes_per_state(Some(MIN_BYTES_PER_STATE))
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) enum Seek {
Everywhere,
Pattern(SeekCode),
}
pub(crate) fn derive(root: &Node, reg: &RegexType, precise: bool) -> Option<Seek> {
if reg.extp.as_ref().is_some_and(|ext| ext.callout_num != 0) {
return None;
}
let utf8: OnigEncoding = &crate::encodings::utf8::ONIG_ENCODING_UTF8;
if reg.enc.name() != utf8.name() || reg.enc.max_enc_len() != utf8.max_enc_len() {
return None;
}
walk_seek(root, reg, precise)
}
fn walk_seek(root: &Node, reg: &RegexType, precise: bool) -> Option<Seek> {
let mut tails = Vec::new();
if precise {
collect_tails(root, &mut tails);
}
let mut walk = Walk {
reg,
lookaheads_kept: 0,
inexact: false,
tails,
precise,
};
let hir = walk.item(root, true);
if matches_everywhere(&hir) {
return Some(Seek::Everywhere);
}
if precise && nfa_states_estimate(&hir) > MAX_NFA_STATES {
return walk_seek(root, reg, false);
}
let exact = !walk.inexact
&& line_starts_exact(&hir)
&& !reg
.options
.intersects(ONIG_OPTION_NOTBOL | ONIG_OPTION_NOTEOL | ONIG_OPTION_FIND_NOT_EMPTY);
let code = SeekCode::encode(&hir, exact)?;
debug_assert_eq!(code.decode(), hir, "the seek code round-trips");
Some(Seek::Pattern(code))
}
#[derive(Clone, PartialEq, Eq)]
pub(crate) struct SeekCode(Box<[u8]>);
mod tag {
pub(super) const EMPTY: u8 = 0;
pub(super) const LITERAL: u8 = 1;
pub(super) const BYTES: u8 = 2;
pub(super) const UNICODE: u8 = 3;
pub(super) const LOOK: u8 = 4;
pub(super) const REPETITION: u8 = 5;
pub(super) const CONCAT: u8 = 6;
pub(super) const ALTERNATION: u8 = 7;
}
impl SeekCode {
#[cfg(feature = "precompiled")]
pub(crate) fn precompiled_bytes(&self) -> &[u8] {
&self.0
}
#[cfg(feature = "precompiled")]
pub(crate) fn from_precompiled(bytes: Vec<u8>) -> Option<Self> {
struct Check<'a> {
bytes: &'a [u8],
nodes: usize,
}
impl Check<'_> {
fn byte(&mut self) -> Option<u8> {
let (&first, rest) = self.bytes.split_first()?;
self.bytes = rest;
Some(first)
}
fn number(&mut self) -> Option<usize> {
let mut value = 0u64;
for shift in (0..35).step_by(7) {
let b = self.byte()?;
value |= u64::from(b & 0x7f) << shift;
if b & 0x80 == 0 {
return usize::try_from(value)
.ok()
.filter(|v| *v <= u32::MAX as usize);
}
}
None
}
fn node(&mut self, depth: usize) -> Option<()> {
self.nodes = self.nodes.checked_sub(1)?;
if depth > 256 {
return None;
}
match self.byte()? {
tag::EMPTY => {}
tag::LITERAL => {
let len = self.number()?;
self.bytes = self.bytes.get(len..)?;
}
kind @ (tag::BYTES | tag::UNICODE) => {
let count = self.number()?;
if count > self.bytes.len() / 2 {
return None;
}
let mut previous = None;
for _ in 0..count {
let (lo, hi) = if kind == tag::BYTES {
(u32::from(self.byte()?), u32::from(self.byte()?))
} else {
let lo = self.number()? as u32;
let hi = self.number()? as u32;
char::from_u32(lo)?;
char::from_u32(hi)?;
(lo, hi)
};
if lo > hi || previous.is_some_and(|p| p >= lo) {
return None;
}
previous = Some(hi);
}
}
tag::LOOK => {
Look::from_repr(self.number()? as u32)?;
}
tag::REPETITION => {
let min = self.number()?;
let max = self.number()?;
if (max != 0 && max - 1 < min) || self.byte()? > 1 {
return None;
}
self.node(depth + 1)?;
}
tag::CONCAT | tag::ALTERNATION => {
let count = self.number()?;
if count > self.nodes {
return None;
}
for _ in 0..count {
self.node(depth + 1)?;
}
}
_ => return None,
}
Some(())
}
}
let mut check = Check {
bytes: &bytes,
nodes: 65_536,
};
if check.byte()? > 1 {
return None;
}
check.node(0)?;
if !check.bytes.is_empty() {
return None;
}
Some(Self(bytes.into_boxed_slice()))
}
fn encode(hir: &Hir, exact: bool) -> Option<SeekCode> {
let mut out = vec![u8::from(exact)];
Self::write(hir, &mut out)?;
Some(SeekCode(out.into_boxed_slice()))
}
pub(crate) fn exact(&self) -> bool {
self.0[0] != 0
}
fn write(hir: &Hir, out: &mut Vec<u8>) -> Option<()> {
match hir.kind() {
HirKind::Empty => out.push(tag::EMPTY),
HirKind::Literal(literal) => {
out.push(tag::LITERAL);
Self::varint(literal.0.len(), out);
out.extend_from_slice(&literal.0);
}
HirKind::Class(Class::Bytes(class)) => {
out.push(tag::BYTES);
Self::varint(class.ranges().len(), out);
for range in class.ranges() {
out.push(range.start());
out.push(range.end());
}
}
HirKind::Class(Class::Unicode(class)) => {
out.push(tag::UNICODE);
Self::varint(class.ranges().len(), out);
for range in class.ranges() {
Self::varint(range.start() as usize, out);
Self::varint(range.end() as usize, out);
}
}
HirKind::Look(look) => {
out.push(tag::LOOK);
Self::varint(look.as_repr() as usize, out);
}
HirKind::Repetition(rep) => {
out.push(tag::REPETITION);
Self::varint(rep.min as usize, out);
Self::varint(rep.max.map_or(0, |max| max as usize + 1), out);
out.push(u8::from(rep.greedy));
Self::write(&rep.sub, out)?;
}
HirKind::Concat(parts) => {
out.push(tag::CONCAT);
Self::varint(parts.len(), out);
for part in parts {
Self::write(part, out)?;
}
}
HirKind::Alternation(branches) => {
out.push(tag::ALTERNATION);
Self::varint(branches.len(), out);
for branch in branches {
Self::write(branch, out)?;
}
}
HirKind::Capture(_) => return None,
}
Some(())
}
fn varint(mut value: usize, out: &mut Vec<u8>) {
while value >= 0x80 {
out.push((value as u8) | 0x80);
value >>= 7;
}
out.push(value as u8);
}
pub(crate) fn decode(&self) -> Hir {
let mut reader = SeekReader {
code: &self.0,
at: 1,
};
let hir = reader.hir();
debug_assert_eq!(reader.at, self.0.len(), "the seek code is read whole");
hir
}
#[cfg(test)]
pub(crate) fn len(&self) -> usize {
self.0.len()
}
}
struct SeekReader<'a> {
code: &'a [u8],
at: usize,
}
impl SeekReader<'_> {
fn byte(&mut self) -> u8 {
let b = self.code[self.at];
self.at += 1;
b
}
fn varint(&mut self) -> usize {
let mut value = 0usize;
let mut shift = 0;
loop {
let b = self.byte();
value |= usize::from(b & 0x7F) << shift;
if b & 0x80 == 0 {
return value;
}
shift += 7;
}
}
fn hir(&mut self) -> Hir {
match self.byte() {
tag::EMPTY => Hir::empty(),
tag::LITERAL => {
let len = self.varint();
let bytes = &self.code[self.at..self.at + len];
self.at += len;
Hir::literal(bytes)
}
tag::BYTES => {
let count = self.varint();
let ranges: Vec<ClassBytesRange> = (0..count)
.map(|_| {
let lo = self.byte();
let hi = self.byte();
ClassBytesRange::new(lo, hi)
})
.collect();
Hir::class(Class::Bytes(ClassBytes::new(ranges)))
}
tag::UNICODE => {
let count = self.varint();
let ranges: Vec<ClassUnicodeRange> = (0..count)
.map(|_| {
let lo = char::from_u32(self.varint() as u32).expect("a code point");
let hi = char::from_u32(self.varint() as u32).expect("a code point");
ClassUnicodeRange::new(lo, hi)
})
.collect();
Hir::class(Class::Unicode(ClassUnicode::new(ranges)))
}
tag::LOOK => Hir::look(Look::from_repr(self.varint() as u32).expect("a look")),
tag::REPETITION => {
let min = self.varint() as u32;
let max = match self.varint() {
0 => None,
max => Some(max as u32 - 1),
};
let greedy = self.byte() != 0;
let sub = Box::new(self.hir());
Hir::repetition(Repetition {
min,
max,
greedy,
sub,
})
}
tag::CONCAT => {
let count = self.varint();
Hir::concat((0..count).map(|_| self.hir()).collect())
}
tag::ALTERNATION => {
let count = self.varint();
Hir::alternation((0..count).map(|_| self.hir()).collect())
}
_ => unreachable!("a seek code tag"),
}
}
}
impl std::fmt::Debug for SeekCode {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_tuple("SeekCode")
.field(&self.decode().to_string())
.finish()
}
}
fn any_char() -> Hir {
byte_class(u128::MAX, true)
}
fn any_char_except_newline() -> Hir {
byte_class(!(1u128 << b'\n'), true)
}
fn placeholder() -> Hir {
Hir::repetition(Repetition {
min: 0,
max: None,
greedy: true,
sub: Box::new(any_char()),
})
}
fn is_placeholder(hir: &Hir) -> bool {
match hir.kind() {
HirKind::Repetition(rep) => rep.min == 0 && rep.max.is_none() && *rep.sub == any_char(),
_ => false,
}
}
fn non_ascii_char() -> Hir {
Hir::concat(vec![
Hir::class(Class::Bytes(ClassBytes::new([ClassBytesRange::new(
0xC2, 0xF4,
)]))),
Hir::repetition(Repetition {
min: 1,
max: Some(3),
greedy: true,
sub: Box::new(Hir::class(Class::Bytes(ClassBytes::new([
ClassBytesRange::new(0x80, 0xBF),
])))),
}),
])
}
fn matches_everywhere(hir: &Hir) -> bool {
let properties = hir.properties();
if properties.minimum_len() == Some(0) && properties.look_set().is_empty() {
return true;
}
let first = match hir.kind() {
HirKind::Concat(subs) => subs.first(),
_ => Some(hir),
};
first.is_some_and(is_placeholder)
}
fn class_exact(ranges: &[(u32, u32)]) -> bool {
ranges
.iter()
.all(|&(lo, hi)| lo > hi || hi < 0x80 || (lo.max(0x80) == 0x80 && hi >= 0x10FFFF))
}
const EXPLICIT_MEMBERS: usize = 16;
fn explicit_class(ranges: &[(u32, u32)]) -> Option<Hir> {
let mut ascii: u128 = 0;
let mut members = Vec::new();
for &(lo, hi) in ranges {
if lo > hi {
continue;
}
if lo <= 0x7F {
ascii |= ascii_mask(lo as u8, hi.min(0x7F) as u8);
}
let lo = lo.max(0x80);
if hi < lo {
continue;
}
let count = (hi - lo) as usize + 1;
if members.len() + count > EXPLICIT_MEMBERS {
return None;
}
for code in lo..=hi {
if let Some(c) = char::from_u32(code) {
members.push(Hir::literal(c.to_string().into_bytes().into_boxed_slice()));
}
}
}
members.insert(0, byte_class(ascii, false));
Some(alternation(members))
}
fn line_starts_exact(hir: &Hir) -> bool {
if !hir.properties().look_set().contains(Look::StartLF) {
return true;
}
match hir.kind() {
HirKind::Concat(parts) => parts.iter().enumerate().all(|(k, part)| {
if matches!(part.kind(), HirKind::Look(Look::StartLF)) {
parts[k + 1..]
.iter()
.map(|rest| rest.properties().minimum_len().unwrap_or(0))
.sum::<usize>()
>= 1
} else {
line_starts_exact(part)
}
}),
HirKind::Alternation(branches) => branches.iter().all(line_starts_exact),
_ => false,
}
}
fn class_hir(ranges: impl IntoIterator<Item = (u32, u32)>, negate: bool) -> Hir {
let mut ascii: u128 = 0;
let mut non_ascii = false;
let mut all_non_ascii = false;
for (lo, hi) in ranges {
if lo > hi {
continue;
}
if lo <= 0x7F {
ascii |= ascii_mask(lo as u8, hi.min(0x7F) as u8);
}
if hi >= 0x80 {
non_ascii = true;
all_non_ascii |= lo <= 0x80 && hi >= 0x10FFFF;
}
}
if negate {
ascii = !ascii;
non_ascii = !all_non_ascii;
}
byte_class(ascii, non_ascii)
}
fn ascii_mask(lo: u8, hi: u8) -> u128 {
debug_assert!(lo <= hi && hi <= 0x7F);
let width = u32::from(hi - lo) + 1;
let run = if width == 128 {
u128::MAX
} else {
(1u128 << width) - 1
};
run << lo
}
fn byte_class(ascii: u128, non_ascii: bool) -> Hir {
let mut ranges = Vec::new();
let mut byte = 0u32;
while byte <= 0x7F {
if ascii & (1u128 << byte) == 0 {
byte += 1;
continue;
}
let lo = byte;
while byte < 0x7F && ascii & (1u128 << (byte + 1)) != 0 {
byte += 1;
}
ranges.push(ClassBytesRange::new(lo as u8, byte as u8));
byte += 1;
}
match (ranges.is_empty(), non_ascii) {
(false, false) => Hir::class(Class::Bytes(ClassBytes::new(ranges))),
(false, true) => alternation(vec![
Hir::class(Class::Bytes(ClassBytes::new(ranges))),
non_ascii_char(),
]),
(true, true) => non_ascii_char(),
(true, false) => Hir::fail(),
}
}
fn is_fail(hir: &Hir) -> bool {
match hir.kind() {
HirKind::Class(Class::Bytes(class)) => class.ranges().is_empty(),
HirKind::Class(Class::Unicode(class)) => class.ranges().is_empty(),
_ => false,
}
}
fn concat(parts: Vec<Hir>) -> Hir {
if parts.iter().any(is_fail) {
Hir::fail()
} else {
Hir::concat(parts)
}
}
fn repetition(mut min: u32, mut max: Option<u32>, sub: Hir, inexact: &mut bool) -> Hir {
let nested = match sub.kind() {
HirKind::Repetition(inner) => Some((inner.min, inner.max, (*inner.sub).clone())),
_ => None,
};
let sub = match nested {
Some((inner_min, inner_max, inner)) => {
min = min.saturating_mul(inner_min);
max = match (max, inner_max) {
(Some(outer), Some(inner)) => Some(outer.saturating_mul(inner)),
_ => None,
};
inner
}
None => sub,
};
if min > MAX_COUNTED_REPEAT || max.is_some_and(|max| max > MAX_COUNTED_REPEAT) {
*inexact = true;
min = min.min(MAX_COUNTED_REPEAT);
max = None;
}
Hir::repetition(Repetition {
min,
max,
greedy: true,
sub: Box::new(sub),
})
}
fn alternation(mut branches: Vec<Hir>) -> Hir {
branches.retain(|branch| !is_fail(branch));
if branches.is_empty() {
Hir::fail()
} else {
Hir::alternation(branches)
}
}
fn code_ranges(bbuf: &BBuf) -> impl Iterator<Item = (u32, u32)> + '_ {
let words = bbuf.data.as_chunks::<4>().0;
let n = words.first().map_or(0, |chunk| u32::from_ne_bytes(*chunk)) as usize;
words[1..]
.as_chunks::<2>()
.0
.iter()
.take(n)
.map(|pair| (u32::from_ne_bytes(pair[0]), u32::from_ne_bytes(pair[1])))
}
fn class_members(cc: &CClassNode) -> impl Iterator<Item = (u32, u32)> + '_ {
let bits = bitset_members(&cc.bs)
.filter(|&member| member < 0x80)
.map(|member| (member as u32, member as u32));
let ranges = cc
.mbuf
.iter()
.flat_map(code_ranges)
.filter_map(|(lo, hi)| (hi >= 0x80).then_some((lo.max(0x80), hi)));
bits.chain(ranges)
}
fn word_class_look(an: &AnchorNode) -> Option<Look> {
let look = match an.anchor_type {
ANCR_LOOK_BEHIND_NOT => Look::WordStartHalfAscii,
ANCR_PREC_READ_NOT => Look::WordEndHalfAscii,
_ => return None,
};
let body = an.body.as_deref()?;
let word_bytes = (b'0'..=b'9')
.chain(b'A'..=b'Z')
.chain(b'a'..=b'z')
.chain(*b"_");
let holds_every_word_byte = match &body.inner {
NodeInner::CType(ct) => ct.ctype == ONIGENC_CTYPE_WORD as i32 && !ct.not,
NodeInner::CClass(cc) => {
let mut members = word_bytes.map(|byte| bitset_at(&cc.bs, byte as usize));
if cc.is_not() {
members.all(|member| !member)
} else {
members.all(|member| member)
}
}
_ => false,
};
holds_every_word_byte.then_some(look)
}
fn is_capture_check(node: &Node) -> bool {
matches!(node.inner, NodeInner::BackRef(_)) && node.has_status(ND_ST_CHECKER)
}
fn automatic_possessive(node: &Node) -> bool {
node.has_status(ND_ST_STRICT_REAL_REPEAT)
}
fn droppable(node: &Node) -> bool {
match &node.inner {
NodeInner::BackRef(_) => is_capture_check(node),
NodeInner::Anchor(an) => match an.anchor_type {
ANCR_PREC_READ_NOT | ANCR_LOOK_BEHIND_NOT => word_class_look(an).is_none(),
ANCR_PREC_READ
| ANCR_LOOK_BEHIND
| ANCR_BEGIN_POSITION
| ANCR_TEXT_SEGMENT_BOUNDARY
| ANCR_NO_TEXT_SEGMENT_BOUNDARY => true,
_ => false,
},
NodeInner::Gimmick(_) => true,
NodeInner::String(sn) => sn.s.is_empty(),
NodeInner::Quant(qn) => qn.upper == 0 || qn.body.as_deref().is_none_or(droppable),
NodeInner::Bag(bn) => match &bn.bag_data {
BagData::IfElse {
then_node,
else_node,
} => {
bn.body.as_deref().is_none_or(droppable)
&& then_node.as_deref().is_none_or(droppable)
&& else_node.as_deref().is_none_or(droppable)
}
_ => bn.body.as_deref().is_none_or(droppable),
},
NodeInner::List(_) | NodeInner::Alt(_) => cons_items(node).into_iter().all(droppable),
_ => false,
}
}
fn class_union(node: &Node, ranges: &mut Vec<(u32, u32)>) -> Option<(usize, usize)> {
match &node.inner {
NodeInner::CClass(cc) => {
if cc.is_not() {
return None;
}
ranges.extend(class_members(cc));
Some((1, 1))
}
NodeInner::String(sn) => {
if node.has_status(ND_ST_IGNORECASE) && !sn.is_crude() {
return None;
}
let text = std::str::from_utf8(&sn.s).ok()?;
let mut count = 0;
for c in text.chars() {
ranges.push((c as u32, c as u32));
count += 1;
}
Some((count, count))
}
NodeInner::List(_) => {
let mut lens = (0, 0);
for item in cons_items(node) {
let (lo, hi) = class_union(item, ranges)?;
lens = (lens.0 + lo, lens.1 + hi);
}
Some(lens)
}
NodeInner::Alt(_) => {
let mut lens = (usize::MAX, 0);
for item in cons_items(node) {
let (lo, hi) = class_union(item, ranges)?;
lens = (lens.0.min(lo), lens.1.max(hi));
}
Some(lens)
}
NodeInner::Bag(bn) if !matches!(bn.bag_data, BagData::IfElse { .. }) => {
class_union(bn.body.as_deref()?, ranges)
}
_ => None,
}
}
fn cons_items(node: &Node) -> Vec<&Node> {
let mut items = Vec::new();
let mut cur = node;
loop {
match &cur.inner {
NodeInner::List(cons) | NodeInner::Alt(cons) => {
items.push(&*cons.car);
match &cons.cdr {
Some(next) => cur = next,
None => break,
}
}
_ => {
items.push(cur);
break;
}
}
}
items
}
fn collect_tails(node: &Node, out: &mut Vec<*const Node>) {
match &node.inner {
NodeInner::List(_) => {
if let Some(&last) = cons_items(node).last() {
collect_tails(last, out);
}
}
NodeInner::Alt(_) => {
for branch in cons_items(node) {
collect_tails(branch, out);
}
}
NodeInner::Bag(bn) if !matches!(bn.bag_data, BagData::IfElse { .. }) => {
if let Some(body) = bn.body.as_deref() {
collect_tails(body, out);
}
}
NodeInner::Anchor(an) if an.anchor_type == ANCR_SEMI_END_BUF => {
out.push(node as *const Node);
}
_ => {}
}
}
struct Walk<'a> {
reg: &'a RegexType,
lookaheads_kept: u32,
inexact: bool,
tails: Vec<*const Node>,
precise: bool,
}
impl Walk<'_> {
fn item(&mut self, node: &Node, rest_droppable: bool) -> Hir {
if let NodeInner::Anchor(an) = &node.inner {
if an.anchor_type == ANCR_PREC_READ {
return self.lookahead(an, rest_droppable);
}
}
self.node(node, rest_droppable)
}
fn lookahead(&mut self, an: &AnchorNode, keep: bool) -> Hir {
self.inexact = true;
if keep {
self.lookaheads_kept += 1;
an.body
.as_deref()
.map_or_else(Hir::empty, |body| self.item(body, true))
} else {
Hir::empty()
}
}
fn node(&mut self, node: &Node, rest_droppable: bool) -> Hir {
if node.has_status(ND_ST_LITERAL_ALT) {
return self.trie(node);
}
match &node.inner {
NodeInner::String(sn) => {
if node.has_status(ND_ST_IGNORECASE) {
self.inexact = true;
if !sn.is_crude() {
return placeholder();
}
}
if sn.s.is_empty() {
Hir::empty()
} else {
Hir::literal(sn.s.clone().into_boxed_slice())
}
}
NodeInner::CClass(cc) => self.cclass(cc),
NodeInner::CType(ct) => self.ctype(node, ct),
NodeInner::BackRef(_) | NodeInner::Call(_) => {
self.inexact = true;
placeholder()
}
NodeInner::Quant(qn) => {
let Some(body) = qn.body.as_deref() else {
return Hir::empty();
};
let body_rest = rest_droppable && (0..=1).contains(&qn.upper);
let sub = self.item(body, body_rest);
let min = u32::try_from(qn.lower.max(0)).unwrap_or(u32::MAX);
let max = (qn.upper >= 0).then_some(qn.upper as u32);
repetition(min, max, sub, &mut self.inexact)
}
NodeInner::Bag(bn) => match &bn.bag_data {
BagData::IfElse {
then_node,
else_node,
} => {
self.inexact = true;
let condition = bn
.body
.as_deref()
.map_or_else(Hir::empty, |cond| self.condition(cond));
let then = then_node
.as_deref()
.map_or_else(Hir::empty, |n| self.item(n, rest_droppable));
let otherwise = else_node
.as_deref()
.map_or_else(Hir::empty, |n| self.item(n, rest_droppable));
alternation(vec![concat(vec![condition, then]), otherwise])
}
BagData::StopBacktrack | BagData::TextSegment => {
if !(matches!(bn.bag_data, BagData::StopBacktrack)
&& automatic_possessive(node))
{
self.inexact = true;
}
bn.body
.as_deref()
.map_or_else(Hir::empty, |body| self.item(body, rest_droppable))
}
_ => bn
.body
.as_deref()
.map_or_else(Hir::empty, |body| self.item(body, rest_droppable)),
},
NodeInner::Anchor(an) => {
if self.tails.contains(&(node as *const Node)) {
let newline = repetition(0, Some(1), Hir::literal(*b"\n"), &mut self.inexact);
return concat(vec![newline, Hir::look(Look::End)]);
}
self.anchor(an)
}
NodeInner::List(_) => self.list(node, rest_droppable),
NodeInner::Alt(_) => {
let branches = cons_items(node);
if !self.precise
&& branches.len() >= 2
&& matches!(branches[0].inner, NodeInner::CClass(_))
{
let mut ranges = Vec::new();
let mut lens: Option<(usize, usize)> = Some((usize::MAX, 0));
for branch in &branches {
match class_union(branch, &mut ranges) {
Some((lo, hi)) => lens = lens.map(|(a, b)| (a.min(lo), b.max(hi))),
None => {
lens = None;
break;
}
}
}
if let Some((min, max)) = lens {
self.inexact = true;
return Hir::repetition(Repetition {
min: u32::try_from(min).unwrap_or(u32::MAX),
max: Some(u32::try_from(max).unwrap_or(u32::MAX)),
greedy: true,
sub: Box::new(class_hir(ranges, false)),
});
}
}
let branches = branches
.into_iter()
.map(|branch| self.item(branch, rest_droppable))
.collect();
alternation(branches)
}
NodeInner::Gimmick(gn) => {
let keep = match gn.gimmick_type {
GimmickType::Save => gn.detail_type == SaveType::Keep as i32,
GimmickType::UpdateVar => {
gn.detail_type == UpdateVarType::KeepFromStackLast as i32
}
_ => false,
};
if !keep {
self.inexact = true;
}
match gn.gimmick_type {
GimmickType::Fail => Hir::fail(),
_ => Hir::empty(),
}
}
}
}
fn condition(&mut self, cond: &Node) -> Hir {
if is_capture_check(cond) {
Hir::empty()
} else {
self.item(cond, false)
}
}
fn list(&mut self, node: &Node, rest_droppable: bool) -> Hir {
let items = cons_items(node);
let mut parts = Vec::with_capacity(items.len());
let mut kept_before = false;
for (k, item) in items.iter().enumerate() {
let rest =
rest_droppable && !kept_before && items[k + 1..].iter().all(|n| droppable(n));
let kept = self.lookaheads_kept;
parts.push(self.item(item, rest));
if self.lookaheads_kept != kept {
kept_before = true;
}
}
concat(parts)
}
fn anchor(&mut self, an: &AnchorNode) -> Hir {
if !matches!(
an.anchor_type,
ANCR_BEGIN_BUF | ANCR_END_BUF | ANCR_END_LINE | ANCR_BEGIN_LINE
) {
self.inexact = true;
}
match an.anchor_type {
ANCR_PREC_READ | ANCR_LOOK_BEHIND => Hir::empty(),
ANCR_PREC_READ_NOT | ANCR_LOOK_BEHIND_NOT => {
word_class_look(an).map_or_else(Hir::empty, Hir::look)
}
ANCR_BEGIN_BUF => Hir::look(Look::Start),
ANCR_END_BUF => Hir::look(Look::End),
ANCR_SEMI_END_BUF | ANCR_END_LINE => Hir::look(Look::EndLF),
ANCR_BEGIN_LINE => Hir::look(Look::StartLF),
ANCR_BEGIN_POSITION => Hir::empty(),
ANCR_WORD_BOUNDARY | ANCR_NO_WORD_BOUNDARY | ANCR_WORD_BEGIN | ANCR_WORD_END
if an.ascii_mode =>
{
Hir::look(match an.anchor_type {
ANCR_WORD_BOUNDARY => Look::WordAscii,
ANCR_NO_WORD_BOUNDARY => Look::WordAsciiNegate,
ANCR_WORD_BEGIN => Look::WordStartAscii,
_ => Look::WordEndAscii,
})
}
ANCR_WORD_BOUNDARY | ANCR_NO_WORD_BOUNDARY | ANCR_WORD_BEGIN | ANCR_WORD_END => {
Hir::alternation(vec![
Hir::look(match an.anchor_type {
ANCR_WORD_BOUNDARY => Look::WordAscii,
ANCR_NO_WORD_BOUNDARY => Look::WordAsciiNegate,
ANCR_WORD_BEGIN => Look::WordStartAscii,
_ => Look::WordEndAscii,
}),
Hir::look(Look::WordStartHalfAscii),
Hir::look(Look::WordEndHalfAscii),
])
}
_ => Hir::empty(),
}
}
fn cclass(&mut self, cc: &CClassNode) -> Hir {
let ranges: Vec<(u32, u32)> = class_members(cc).collect();
if self.precise && !cc.is_not() {
if let Some(hir) = explicit_class(&ranges) {
return hir;
}
}
if !class_exact(&ranges) {
self.inexact = true;
}
class_hir(ranges, cc.is_not())
}
fn ctype(&mut self, node: &Node, ct: &CtypeNode) -> Hir {
if ct.ctype == CTYPE_ANYCHAR {
return if node.has_status(ND_ST_MULTILINE) {
any_char()
} else {
any_char_except_newline()
};
}
if !(ct.ctype == ONIGENC_CTYPE_WORD as i32 && ct.ascii_mode) {
self.inexact = true;
}
if ct.ctype == ONIGENC_CTYPE_WORD as i32 {
let word = ascii_mask(b'0', b'9')
| ascii_mask(b'A', b'Z')
| ascii_mask(b'_', b'_')
| ascii_mask(b'a', b'z');
return if ct.not {
byte_class(!word, true)
} else {
byte_class(word, !ct.ascii_mode)
};
}
any_char()
}
fn trie(&mut self, node: &Node) -> Hir {
let reg = self.reg;
let Some(trie) = literal_alt_trie_index(node).and_then(|idx| reg.literal_tries.get(idx))
else {
self.inexact = true;
return placeholder();
};
let literals = trie.literals();
if !trie.is_case_insensitive() {
return alternation(
literals
.iter()
.map(|literal| Hir::literal(literal.clone().into_boxed_slice()))
.collect(),
);
}
self.inexact = true;
let Some(folds) = trie.folds() else {
return placeholder();
};
let mut branches: Vec<Hir> = literals
.iter()
.map(|literal| {
concat(
literal
.iter()
.map(|&c| {
if c.is_ascii_alphabetic() {
Hir::class(Class::Bytes(ClassBytes::new([
ClassBytesRange::new(
c.to_ascii_lowercase(),
c.to_ascii_lowercase(),
),
ClassBytesRange::new(
c.to_ascii_uppercase(),
c.to_ascii_uppercase(),
),
])))
} else {
Hir::literal([c])
}
})
.collect(),
)
})
.collect();
let mut non_ascii: Vec<Hir> = folds
.class_members
.iter()
.filter_map(|&(code, _)| char::from_u32(code))
.map(|c| Hir::literal(c.to_string().into_bytes().into_boxed_slice()))
.chain(
folds
.ligatures
.iter()
.map(|(bytes, _)| Hir::literal(bytes.clone().into_boxed_slice())),
)
.collect();
if !non_ascii.is_empty() {
non_ascii.sort_by_key(|a| a.to_string());
non_ascii.dedup_by(|a, b| a.to_string() == b.to_string());
let longest = literals.iter().map(Vec::len).max().unwrap_or(1).max(1);
let rest = u32::try_from(longest - 1).unwrap_or(u32::MAX);
let ascii_prefix = repetition(
0,
Some(rest),
Hir::class(Class::Bytes(ClassBytes::new([ClassBytesRange::new(
0x00, 0x7F,
)]))),
&mut self.inexact,
);
let suffix = repetition(0, Some(rest), any_char(), &mut self.inexact);
branches.push(concat(vec![ascii_prefix, alternation(non_ascii), suffix]));
}
alternation(branches)
}
}
pub(crate) struct Automata {
search: Arc<hybrid_regex::Regex>,
dfa: hybrid::DFA,
entries: Vec<u16>,
own: Vec<u16>,
long: Vec<u16>,
settle: usize,
#[cfg(feature = "scanner-group-prototype")]
frozen: Option<Arc<group_archive::Frozen>>,
#[cfg(feature = "precompiled-eager")]
portable: Option<Arc<portable::Portable>>,
}
#[derive(Clone, Default)]
pub(crate) struct AutomataCell(Arc<OnceLock<Option<Arc<Automata>>>>);
impl AutomataCell {
pub(crate) fn new() -> Self {
Self::default()
}
pub(crate) fn get_or_build(
&self,
build: impl FnOnce() -> Option<Automata>,
) -> Option<Arc<Automata>> {
self.0.get_or_init(|| build().map(Arc::new)).clone()
}
}
pub(crate) struct SetPrefilter {
search: Arc<hybrid_regex::Regex>,
search_cache: hybrid_regex::Cache,
dfa: hybrid::DFA,
dfa_cache: hybrid::Cache,
patset: PatternBits,
entries: Vec<u16>,
own: Vec<u16>,
long: Vec<u16>,
settle: usize,
ask_early: bool,
gave_up: bool,
walked: usize,
dispatch: Vec<[u64; 4]>,
long_dispatch: [u64; 4],
any_dispatch: [u64; 4],
admissible: Vec<u32>,
stamp: u32,
candidates: Vec<u16>,
walked_at: Option<usize>,
search_memo: Option<SearchMemo>,
candidate_memo: CandidateMemo,
#[cfg(feature = "scanner-group-prototype")]
frozen: Option<Arc<group_archive::Frozen>>,
#[cfg(feature = "precompiled-eager")]
portable: Option<(Arc<portable::Portable>, portable::Caches)>,
}
#[derive(Default)]
struct CandidateMemo {
subject: Option<(crate::regset::FallbackMemoIdentity, usize)>,
active: bool,
last_start: usize,
at: Vec<u32>,
spans: Vec<(u32, u16, bool)>,
store: Vec<u16>,
}
#[inline(never)]
fn record_match(
dfa: &hybrid::DFA,
cache: &hybrid::Cache,
patset: &mut PatternBits,
sid: LazyStateID,
) {
for k in 0..dfa.match_len(cache, sid) {
patset.insert(dfa.match_pattern(cache, sid, k));
}
}
#[inline(never)]
fn record_needed(
dfa: &hybrid::DFA,
cache: &hybrid::Cache,
patset: &mut PatternBits,
sid: LazyStateID,
stamps: &[u32],
stamp: u32,
) -> usize {
let mut found = 0;
for k in 0..dfa.match_len(cache, sid) {
let pid = dfa.match_pattern(cache, sid, k);
if patset.insert(pid) && stamps[pid.as_usize()] == stamp {
found += 1;
}
}
found
}
struct PatternBits {
words: Box<[u64]>,
len: usize,
capacity: usize,
}
impl PatternBits {
fn new(capacity: usize) -> Self {
PatternBits {
words: vec![0; capacity.div_ceil(64)].into_boxed_slice(),
len: 0,
capacity,
}
}
#[inline]
fn clear(&mut self) {
if self.len > 0 {
self.words.fill(0);
self.len = 0;
}
}
#[inline]
fn contains(&self, pid: PatternID) -> bool {
let i = pid.as_usize();
i < self.capacity && (self.words[i / 64] >> (i % 64)) & 1 == 1
}
#[inline]
fn insert(&mut self, pid: PatternID) -> bool {
let i = pid.as_usize();
assert!(i < self.capacity, "pattern ID {i} outside the set");
let word = &mut self.words[i / 64];
let bit = 1u64 << (i % 64);
if *word & bit != 0 {
return false;
}
*word |= bit;
self.len += 1;
true
}
#[inline]
fn is_empty(&self) -> bool {
self.len == 0
}
#[inline]
fn is_full(&self) -> bool {
self.len == self.capacity
}
#[inline]
fn iter(&self) -> impl Iterator<Item = usize> + '_ {
self.words.iter().enumerate().flat_map(|(w, &word)| {
let mut bits = word;
std::iter::from_fn(move || {
(bits != 0).then(|| {
let b = bits.trailing_zeros() as usize;
bits &= bits - 1;
w * 64 + b
})
})
})
}
}
const CANDIDATE_MEMO_POSITIONS: usize = 4096;
impl CandidateMemo {
#[inline]
fn begin(
&mut self,
subject: Option<(crate::regset::FallbackMemoIdentity, usize)>,
start: usize,
) {
if subject.is_some() && subject == self.subject {
if start <= self.last_start {
self.active = true;
}
self.last_start = start;
return;
}
self.subject = subject;
self.active = false;
self.last_start = start;
self.at.clear();
self.spans.clear();
self.store.clear();
}
fn get(&self, at: usize) -> Option<(&[u16], bool)> {
let at = u32::try_from(at).ok()?;
let i = self.at.binary_search(&at).ok()?;
let (start, len, some) = self.spans[i];
Some((
&self.store[start as usize..start as usize + len as usize],
some,
))
}
fn insert(&mut self, at: usize, candidates: &[u16], some: bool) {
if self.at.len() >= CANDIDATE_MEMO_POSITIONS {
return;
}
let (Ok(at), Ok(len)) = (u32::try_from(at), u16::try_from(candidates.len())) else {
return;
};
let start = self.store.len() as u32;
self.store.extend_from_slice(candidates);
let i = match self.at.binary_search(&at) {
Ok(_) => return,
Err(i) => i,
};
self.at.insert(i, at);
self.spans.insert(i, (start, len, some));
}
}
#[derive(Clone, Copy)]
struct SearchMemo {
subject: (crate::regset::FallbackMemoIdentity, usize),
from: usize,
until: usize,
at: Option<usize>,
}
#[derive(Clone, Copy, PartialEq, Eq)]
enum ScanEnd {
Finished,
Settled,
Bound,
GaveUp,
}
impl std::fmt::Debug for SetPrefilter {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("SetPrefilter")
.field("entries", &self.entries.len())
.field("own", &self.own.len())
.field("nfa_states", &self.nfa_states())
.finish_non_exhaustive()
}
}
impl Automata {
pub(crate) fn build<'a>(
entries: impl Iterator<Item = (Option<&'a Seek>, &'a RegexType)>,
skip_tiny: bool,
) -> Option<Self> {
#[cfg(feature = "scanner-load-profile")]
let _phase =
crate::scanner::load_profile::scope(crate::scanner::load_profile::Phase::GroupBuild);
let mut hirs: Vec<Hir> = Vec::new();
let mut covered = Vec::new();
let mut own = Vec::new();
for (index, (seek, reg)) in entries.enumerate() {
let index = u16::try_from(index).ok()?;
match seek {
Some(Seek::Pattern(code)) => {
hirs.push(code.decode());
covered.push(index);
}
Some(Seek::Everywhere) => own.push(index),
None if reg.extp.as_ref().is_some_and(|ext| ext.callout_num != 0) => return None,
None => own.push(index),
}
}
if covered.is_empty() {
return None;
}
let entries = covered;
let nfa = thompson::Compiler::new()
.configure(
thompson::Config::new()
.utf8(false)
.which_captures(thompson::WhichCaptures::None)
.nfa_size_limit(Some(NFA_SIZE_LIMIT)),
)
.build_many_from_hir(&hirs)
.ok()?;
if nfa.states().len() > MAX_NFA_STATES {
return None;
}
if skip_tiny
&& entries.len() <= TINY_SET_ENTRIES
&& nfa.states().len() <= TINY_SET_NFA_STATES
{
return None;
}
let nfarev = thompson::Compiler::new()
.configure(
thompson::Config::new()
.utf8(false)
.reverse(true)
.which_captures(thompson::WhichCaptures::None)
.nfa_size_limit(Some(NFA_SIZE_LIMIT)),
)
.build_many_from_hir(&hirs)
.ok()?;
let forward = search_dfa(nfa.clone(), MatchKind::LeftmostFirst)?;
let reverse = search_dfa(nfarev, MatchKind::All)?;
let search = Arc::new(hybrid_regex::Builder::new().build_from_dfas(forward, reverse));
let dfa = hybrid::Builder::new()
.configure(overlapping_config())
.build_from_nfa(nfa)
.ok()?;
let mut long = Vec::new();
let mut settle = 0;
for (pid, hir) in hirs.iter().enumerate() {
match hir.properties().maximum_len() {
Some(len) if len < LONG_SEEK_BYTES => settle = settle.max(len + 1),
_ => long.push(pid as u16),
}
}
Some(Automata {
search,
dfa,
entries,
own,
long,
settle,
#[cfg(feature = "scanner-group-prototype")]
frozen: None,
#[cfg(feature = "precompiled-eager")]
portable: None,
})
}
pub(crate) fn nfa_states(&self) -> usize {
#[cfg(feature = "scanner-group-prototype")]
if let Some(frozen) = &self.frozen {
return frozen.nfa_states;
}
#[cfg(feature = "precompiled-eager")]
if let Some(portable) = &self.portable {
return portable.nfa_states();
}
self.dfa.get_nfa().states().len()
}
pub(crate) fn memory_usage(&self) -> usize {
let bytes = automata_memory_usage(&self.dfa, &self.search);
#[cfg(feature = "precompiled-eager")]
let bytes = bytes + self.portable.as_ref().map_or(0, |p| p.memory_usage());
bytes
}
}
fn automata_memory_usage(dfa: &hybrid::DFA, search: &hybrid_regex::Regex) -> usize {
dfa.get_nfa().memory_usage()
+ search.reverse().get_nfa().memory_usage()
+ search.forward().memory_usage()
+ search.reverse().memory_usage()
+ dfa.memory_usage()
}
fn search_dfa(nfa: thompson::NFA, match_kind: MatchKind) -> Option<hybrid::DFA> {
hybrid::Builder::new()
.configure(
hybrid::Config::new()
.match_kind(match_kind)
.cache_capacity(SEARCH_CACHE_CAPACITY)
.minimum_cache_clear_count(Some(MIN_CACHE_CLEARS))
.minimum_bytes_per_state(Some(SEARCH_MIN_BYTES_PER_STATE)),
)
.build_from_nfa(nfa)
.ok()
}
impl SetPrefilter {
pub(crate) fn new(automata: &Automata) -> Self {
#[cfg(feature = "scanner-load-profile")]
let _phase = crate::scanner::load_profile::scope(
crate::scanner::load_profile::Phase::ScannerScratch,
);
let search_cache = automata.search.create_cache();
let dfa_cache = automata.dfa.create_cache();
let covered = automata.entries.len();
SetPrefilter {
search: automata.search.clone(),
search_cache,
dfa: automata.dfa.clone(),
dfa_cache,
patset: PatternBits::new(covered),
entries: automata.entries.clone(),
own: automata.own.clone(),
long: automata.long.clone(),
settle: automata.settle,
ask_early: false,
gave_up: false,
walked: 0,
dispatch: vec![[u64::MAX; 4]; covered],
long_dispatch: [0; 4],
any_dispatch: [0; 4],
admissible: vec![0; covered],
stamp: 0,
candidates: Vec::new(),
walked_at: None,
search_memo: None,
candidate_memo: CandidateMemo::default(),
#[cfg(feature = "scanner-group-prototype")]
frozen: automata.frozen.clone(),
#[cfg(feature = "precompiled-eager")]
portable: automata.portable.as_ref().map(|p| (p.clone(), p.caches())),
}
}
pub(crate) fn set_dispatch(&mut self, mut bytes: impl FnMut(u16) -> [u64; 4]) {
for (pid, &index) in self.entries.iter().enumerate() {
self.dispatch[pid] = bytes(index);
}
let union = |pids: &mut dyn Iterator<Item = usize>| {
pids.fold([0u64; 4], |mut bits, pid| {
for (word, dispatch) in bits.iter_mut().zip(self.dispatch[pid]) {
*word |= dispatch;
}
bits
})
};
let long_dispatch = union(&mut self.long.iter().map(|&pid| pid as usize));
let any_dispatch = union(&mut (0..self.entries.len()));
self.long_dispatch = long_dispatch;
self.any_dispatch = any_dispatch;
}
#[inline]
pub(crate) fn any_dispatch(&self) -> [u64; 4] {
self.any_dispatch
}
pub(crate) fn own(&self) -> &[u16] {
&self.own
}
pub(crate) fn covered_entries(&self) -> &[u16] {
&self.entries
}
pub(crate) fn covered(&self) -> usize {
self.entries.len()
}
pub(crate) fn candidate_in(
&mut self,
haystack: &[u8],
from: usize,
until: usize,
subject: Option<(crate::regset::FallbackMemoIdentity, usize)>,
admits: &mut dyn FnMut(u16, usize) -> bool,
) -> Option<usize> {
let mut search_from = from;
let mut memo_from = from;
if let (Some(subject), Some(memo)) = (subject, self.search_memo) {
if memo.subject == subject && from >= memo.from {
match memo.at {
Some(at) if from <= at => return (at < until).then_some(at),
None if until <= memo.until => return None,
None if from <= memo.until => {
search_from = memo.until;
memo_from = memo.from;
}
_ => {}
}
}
}
let at = self.search_in(haystack, search_from, until, admits);
if let Some(subject) = subject {
self.search_memo = Some(SearchMemo {
subject,
from: memo_from,
until,
at,
});
}
at
}
#[inline]
pub(crate) fn note_position(
&mut self,
subject: Option<(crate::regset::FallbackMemoIdentity, usize)>,
at: usize,
candidate: bool,
next: usize,
) {
if let (Some(subject), Some(memo)) = (subject, self.search_memo.as_mut()) {
if memo.subject == subject && memo.at.is_none() && memo.until == at {
if candidate {
memo.at = Some(at);
} else {
memo.until = next;
}
}
}
}
fn search_in(
&mut self,
haystack: &[u8],
from: usize,
until: usize,
admits: &mut dyn FnMut(u16, usize) -> bool,
) -> Option<usize> {
if from >= until || from > haystack.len() {
return None;
}
let mut next = from;
if !self.long.is_empty() {
if self
.candidates_at(haystack, from, admits)
.is_some_and(|candidates| !candidates.is_empty())
{
return Some(from);
}
next = from + 1;
while next < haystack.len() && (haystack[next] & 0xC0) == 0x80 {
next += 1;
}
}
let span_end = haystack.len().min(until.saturating_add(self.settle));
let found = if next <= span_end {
#[cfg(test)]
EARLIEST_CALLS.with(|calls| calls.set(calls.get() + 1));
let mut ordinary = || {
let input = Input::new(haystack).span(next..span_end);
#[cfg(feature = "scanner-group-prototype")]
if let Some(frozen) = &self.frozen {
return frozen.search.try_search(&input);
}
self.search.try_search(&mut self.search_cache, &input)
};
#[cfg(feature = "precompiled-eager")]
let result = if let Some((p, c)) = &mut self.portable {
Ok(p.search(c, haystack, next, span_end))
} else {
ordinary()
};
#[cfg(not(feature = "precompiled-eager"))]
let result = ordinary();
let Ok(found) = result else {
self.gave_up = true;
return Some(from);
};
#[cfg(test)]
META_BYTES.with(|bytes| {
let read = found.map_or(span_end - next, |m| {
(m.end() - next) + (m.end() - m.start())
});
bytes.set(bytes.get() + read as u64);
});
found.map(|m| m.start()).filter(|&at| at < until)
} else {
None
};
if self.long.is_empty() {
return found;
}
let dispatch = self.long_dispatch;
let limit = found.unwrap_or(until);
let mut p = next;
while p < limit {
let admitted = haystack
.get(p)
.is_none_or(|&byte| (dispatch[byte as usize / 64] >> (byte % 64)) & 1 == 1);
if admitted
&& self
.candidates_at(haystack, p, admits)
.is_some_and(|candidates| !candidates.is_empty())
{
return Some(p);
}
p += 1;
while p < limit && p < haystack.len() && (haystack[p] & 0xC0) == 0x80 {
p += 1;
}
}
found
}
#[inline]
pub(crate) fn candidates_at(
&mut self,
haystack: &[u8],
at: usize,
admits: &mut dyn FnMut(u16, usize) -> bool,
) -> Option<&[u16]> {
self.candidates.clear();
self.walked_at = Some(at);
if self.candidate_memo.active {
if let Some((memo, some)) = self.candidate_memo.get(at) {
self.candidates.extend_from_slice(memo);
return some.then_some(&self.candidates);
}
}
self.patset.clear();
let (end, asked) = self.scan_candidates(haystack, at, admits);
self.ask_early = end == ScanEnd::Bound || (asked && end == ScanEnd::Settled);
if end == ScanEnd::Finished && self.patset.is_empty() {
if self.candidate_memo.active {
self.candidate_memo.insert(at, &[], false);
}
return None;
}
let entries = &self.entries;
if end == ScanEnd::Bound {
let stamps = &self.admissible;
let stamp = self.stamp;
let patset = &self.patset;
self.candidates.extend(
entries
.iter()
.enumerate()
.filter(|&(pid, _)| {
!asked
|| stamps[pid] == stamp
|| patset.contains(PatternID::new_unchecked(pid))
})
.map(|(_, &index)| index),
);
} else {
self.candidates
.extend(self.patset.iter().map(|pid| entries[pid]));
}
if self.candidate_memo.active {
self.candidate_memo.insert(at, &self.candidates, true);
}
Some(&self.candidates)
}
#[inline]
pub(crate) fn candidates(&self) -> &[u16] {
&self.candidates
}
#[inline]
pub(crate) fn walked_at(&self) -> Option<usize> {
self.walked_at
}
#[inline]
pub(crate) fn begin_search(
&mut self,
subject: Option<(crate::regset::FallbackMemoIdentity, usize)>,
start: usize,
) {
self.walked_at = None;
self.candidate_memo.begin(subject, start);
self.walked = 0;
self.dfa_cache.search_start(0);
}
fn scan_candidates(
&mut self,
haystack: &[u8],
at: usize,
admits: &mut dyn FnMut(u16, usize) -> bool,
) -> (ScanEnd, bool) {
#[cfg(feature = "precompiled-eager")]
if let Some((p, c)) = &mut self.portable {
return p.scan(
c,
&mut self.patset,
&self.entries,
&self.long,
self.settle,
self.ask_early,
&self.dispatch,
&mut self.admissible,
&mut self.stamp,
haystack,
at,
admits,
);
}
#[cfg(feature = "scanner-group-prototype")]
if let Some(frozen) = &self.frozen {
return frozen.scan(
&mut self.patset,
&self.entries,
&self.long,
self.settle,
self.ask_early,
&self.dispatch,
&mut self.admissible,
&mut self.stamp,
haystack,
at,
admits,
);
}
let Self {
dfa,
dfa_cache: cache,
patset,
entries,
long,
settle,
ask_early,
gave_up,
walked,
dispatch,
admissible: stamps,
stamp,
..
} = self;
let mut i = at;
let (end, asked) = Self::walk(
dfa, cache, patset, entries, long, *settle, *ask_early, dispatch, stamps, stamp,
haystack, at, &mut i, admits,
);
*walked += i - at;
cache.search_update(*walked);
if end == ScanEnd::GaveUp {
*gave_up = true;
return (ScanEnd::Bound, asked);
}
(end, asked)
}
#[allow(clippy::too_many_arguments)]
fn walk(
dfa: &hybrid::DFA,
cache: &mut hybrid::Cache,
patset: &mut PatternBits,
entries: &[u16],
long: &[u16],
settle: usize,
ask_early: bool,
dispatch: &[[u64; 4]],
stamps: &mut [u32],
stamp: &mut u32,
haystack: &[u8],
at: usize,
i: &mut usize,
admits: &mut dyn FnMut(u16, usize) -> bool,
) -> (ScanEnd, bool) {
let end = haystack.len();
let input = Input::new(haystack).span(at..end).anchored(Anchored::Yes);
let Ok(mut sid) = dfa.start_state_forward(cache, &input) else {
return (ScanEnd::GaveUp, false);
};
let cut = end.min(at.saturating_add(CANDIDATE_SCAN_BYTES));
let ask_at = if ask_early {
settle
} else {
settle.max(LONG_WALK_BYTES)
};
let ask_i = at.saturating_add(ask_at);
let mut recorded: Option<(LazyStateID, usize)> = None;
macro_rules! at_end {
($asked:expr) => {
if let Ok(eoi) = dfa.next_eoi_state(cache, sid) {
if eoi.is_match() {
record_match(dfa, cache, patset, eoi);
}
return (ScanEnd::Finished, $asked);
}
return (ScanEnd::GaveUp, $asked);
};
}
let plain_end = cut.min(ask_i);
loop {
if sid.is_dead() {
return (ScanEnd::Finished, false);
}
if sid.is_quit() {
return (ScanEnd::Bound, false);
}
if sid.is_match() && recorded != Some((sid, cache.clear_count())) {
recorded = Some((sid, cache.clear_count()));
record_match(dfa, cache, patset, sid);
if patset.is_full() {
return (ScanEnd::Settled, false);
}
}
if *i >= plain_end {
if *i == end {
at_end!(false);
}
if *i == cut {
return (ScanEnd::Bound, false);
}
break;
}
#[cfg(test)]
SCAN_STEPS.with(|steps| steps.set(steps.get() + 1));
sid = match dfa.next_state(cache, sid, haystack[*i]) {
Ok(sid) => sid,
Err(_) => return (ScanEnd::GaveUp, false),
};
*i += 1;
}
*stamp = stamp.wrapping_add(1);
if *stamp == 0 {
stamps.fill(0);
*stamp = 1;
}
let byte = haystack[at];
let mut needed = 0usize;
for &pid in long.iter() {
let bits = dispatch[pid as usize];
if patset.contains(PatternID::new_unchecked(pid as usize))
|| (bits[byte as usize / 64] >> (byte % 64)) & 1 == 0
{
continue;
}
if admits(entries[pid as usize], at) {
stamps[pid as usize] = *stamp;
needed += 1;
}
}
if needed == 0 {
return (ScanEnd::Settled, true);
}
loop {
#[cfg(test)]
SCAN_STEPS.with(|steps| steps.set(steps.get() + 1));
sid = match dfa.next_state(cache, sid, haystack[*i]) {
Ok(sid) => sid,
Err(_) => return (ScanEnd::GaveUp, true),
};
*i += 1;
if sid.is_dead() {
return (ScanEnd::Finished, true);
}
if sid.is_quit() {
return (ScanEnd::Bound, true);
}
if sid.is_match() && recorded != Some((sid, cache.clear_count())) {
recorded = Some((sid, cache.clear_count()));
needed -= record_needed(dfa, cache, patset, sid, stamps, *stamp);
if needed == 0 || patset.is_full() {
return (ScanEnd::Settled, true);
}
}
if *i == cut {
if *i == end {
at_end!(true);
}
return (ScanEnd::Bound, true);
}
}
}
pub(crate) fn gave_up(&self) -> bool {
self.gave_up
}
pub(crate) fn nfa_states(&self) -> usize {
#[cfg(feature = "scanner-group-prototype")]
if let Some(frozen) = &self.frozen {
return frozen.nfa_states;
}
#[cfg(feature = "precompiled-eager")]
if let Some((portable, _)) = &self.portable {
return portable.nfa_states();
}
self.dfa.get_nfa().states().len()
}
pub(crate) fn memory_usage(&self) -> usize {
let bytes = automata_memory_usage(&self.dfa, &self.search) + self.cache_memory_usage();
#[cfg(feature = "scanner-group-prototype")]
let bytes = bytes + self.frozen.as_ref().map_or(0, |f| f.memory_usage());
#[cfg(feature = "precompiled-eager")]
let bytes = bytes + self.portable.as_ref().map_or(0, |(p, _)| p.memory_usage());
bytes
}
pub(crate) fn cache_memory_usage(&self) -> usize {
let bytes = self.search_cache.memory_usage() + self.dfa_cache.memory_usage();
#[cfg(feature = "precompiled-eager")]
let bytes = bytes + self.portable.as_ref().map_or(0, |(_, c)| c.memory_usage());
bytes
}
pub(crate) fn cache_clears(&self) -> usize {
#[cfg(feature = "precompiled-eager")]
if let Some((_, c)) = &self.portable {
return c.clears();
}
self.dfa_cache.clear_count()
}
}
pub(crate) struct SingleSeek {
dfa: hybrid::DFA,
accept: Option<(hybrid::DFA, accept::AcceptRejects)>,
caches: Pool<SingleCaches, CacheMaker>,
exact: bool,
adaptive: Option<Box<SingleAdaptive>>,
}
struct SingleAdaptive {
seek: Option<AdaptiveDfa>,
accept: Option<AdaptiveDfa>,
}
struct SingleTable {
transitions: Vec<u32>,
classes: [u8; 256],
starts: [u32; 2],
eoi: usize,
}
impl SingleTable {
fn supports(nfa: &thompson::NFA) -> bool {
use regex_automata::util::look::Look;
nfa.states().len() <= MAX_SINGLE_BITSET_NFA_STATES
&& !nfa.is_utf8()
&& nfa
.look_set_any()
.iter()
.all(|look| matches!(look, Look::Start | Look::End))
}
fn closure(nfa: &thompson::NFA, mut pending: u64, start: bool, end: bool) -> u64 {
use regex_automata::{nfa::thompson::State, util::look::Look};
let mut seen = 0u64;
let mut frontier = 0u64;
while pending != 0 {
let index = pending.trailing_zeros() as usize;
let bit = 1u64 << index;
pending &= !bit;
if seen & bit != 0 {
continue;
}
seen |= bit;
match &nfa.states()[index] {
State::Look {
look: Look::Start,
next,
} if start => pending |= 1u64 << next.as_usize(),
State::Look {
look: Look::End,
next,
} if end => pending |= 1u64 << next.as_usize(),
State::Look {
look: Look::End, ..
} => frontier |= bit,
State::Look { .. } | State::Fail => {}
State::Union { alternates } => {
for next in alternates {
pending |= 1u64 << next.as_usize();
}
}
State::BinaryUnion { alt1, alt2 } => {
pending |= (1u64 << alt1.as_usize()) | (1u64 << alt2.as_usize())
}
State::Capture { next, .. } => pending |= 1u64 << next.as_usize(),
State::ByteRange { .. }
| State::Sparse(_)
| State::Dense(_)
| State::Match { .. } => frontier |= bit,
}
}
frontier
}
fn build(nfa: &thompson::NFA, limits: SingleDenseLimits) -> Option<Self> {
use regex_automata::nfa::thompson::State;
if !Self::supports(nfa) {
return None;
}
let classes = nfa.byte_classes();
let stride = classes.alphabet_len();
let max_rows = limits.dfa_bytes.checked_sub(std::mem::size_of::<Self>())?
/ 2
/ (stride * std::mem::size_of::<u32>());
let max_rows = max_rows.min(
limits
.determinize_bytes
.checked_sub(64 * std::mem::size_of::<u64>())?
/ std::mem::size_of::<(u64, bool)>(),
);
if max_rows == 0 {
return None;
}
let mut closures = [0; 64];
let mut matches = 0u64;
for (i, state) in nfa.states().iter().enumerate() {
closures[i] = Self::closure(nfa, 1u64 << i, false, false);
if matches!(state, State::Match { .. }) {
matches |= 1u64 << i;
}
}
let mut states = Vec::with_capacity(max_rows);
let mut transitions = Vec::with_capacity(max_rows * stride);
fn intern(
mask: u64,
at_start: bool,
matches: u64,
states: &mut Vec<(u64, bool)>,
stride: usize,
max_rows: usize,
) -> Option<u32> {
if mask & matches != 0 {
return Some(1);
}
if mask == 0 {
return Some(0);
}
let node = (mask, at_start);
let index = match states.iter().position(|&known| known == node) {
Some(index) => index,
None if states.len() < max_rows => {
states.push(node);
states.len() - 1
}
None => return None,
};
u32::try_from(index * stride + 2).ok()
}
let root = 1u64 << nfa.start_unanchored().as_usize();
let mut starts = [0; 2];
for (i, slot) in starts.iter_mut().enumerate() {
*slot = intern(
Self::closure(nfa, root, i == 1, false),
i == 1,
matches,
&mut states,
stride,
max_rows,
)?;
}
let mut row = 0;
while row < states.len() {
let (mask, at_start) = states[row];
for unit in classes.representatives(..) {
let next = if let Some(byte) = unit.as_u8() {
let mut pending = mask;
let mut next = 0u64;
while pending != 0 {
let i = pending.trailing_zeros() as usize;
pending &= pending - 1;
let target = match &nfa.states()[i] {
State::ByteRange { trans } if trans.matches_byte(byte) => {
Some(trans.next)
}
State::Sparse(trans) => trans.matches_byte(byte),
State::Dense(trans) => trans.matches_byte(byte),
_ => None,
};
if let Some(target) = target {
next |= closures[target.as_usize()];
}
}
intern(next, false, matches, &mut states, stride, max_rows)?
} else {
u32::from(Self::closure(nfa, mask, at_start, true) & matches != 0)
};
transitions.push(next);
}
row += 1;
}
let mut map = [0; 256];
for (byte, slot) in map.iter_mut().enumerate() {
*slot = classes.get(byte as u8);
}
transitions.shrink_to_fit();
Some(Self {
transitions,
classes: map,
starts,
eoi: stride - 1,
})
}
fn search(&self, text: &[u8], start: usize) -> bool {
let mut state = self.starts[usize::from(start == 0)];
for &byte in &text[start..] {
if state <= 1 {
return state == 1;
}
state =
self.transitions[state as usize - 2 + usize::from(self.classes[usize::from(byte)])];
}
if state <= 1 {
return state == 1;
}
self.transitions[state as usize - 2 + self.eoi] == 1
}
fn memory_usage(&self) -> usize {
self.transitions.capacity() * std::mem::size_of::<u32>()
}
}
struct AdaptiveDfa {
limits: SingleDenseLimits,
threshold: usize,
consultations: AtomicUsize,
promotion_attempted: AtomicBool,
dense_decision: OnceLock<Option<Box<SingleTable>>>,
}
impl AdaptiveDfa {
fn new(limits: SingleDenseLimits, threshold: usize) -> Self {
Self {
limits,
threshold,
consultations: AtomicUsize::new(0),
promotion_attempted: AtomicBool::new(false),
dense_decision: OnceLock::new(),
}
}
fn dense(&self) -> Option<&SingleTable> {
self.dense_decision.get().and_then(Option::as_deref)
}
fn initialize_dense_with(&self, build: impl FnOnce() -> Option<Box<SingleTable>>) {
if self
.promotion_attempted
.compare_exchange(false, true, Ordering::Relaxed, Ordering::Relaxed)
.is_ok()
{
let _ = self.dense_decision.get_or_init(build);
}
}
fn record_hybrid_consultation(&self, nfa: &thompson::NFA) {
if self.promotion_attempted.load(Ordering::Relaxed) {
return;
}
let mut count = self.consultations.load(Ordering::Relaxed);
while count < self.threshold {
match self.consultations.compare_exchange_weak(
count,
count + 1,
Ordering::Relaxed,
Ordering::Relaxed,
) {
Ok(_) => {
count += 1;
break;
}
Err(observed) => count = observed,
}
}
if count >= self.threshold {
self.initialize_dense_with(|| SingleTable::build(nfa, self.limits).map(Box::new));
}
}
#[cfg(test)]
fn consultation_count(&self) -> usize {
self.consultations.load(Ordering::Relaxed)
}
#[cfg(test)]
fn promotion_attempted(&self) -> bool {
self.promotion_attempted.load(Ordering::Relaxed)
}
}
#[derive(Clone, Copy)]
struct SingleDenseLimits {
dfa_bytes: usize,
determinize_bytes: usize,
}
const PREFERRED_SINGLE_DENSE_LIMITS: SingleDenseLimits = SingleDenseLimits {
dfa_bytes: SINGLE_DENSE_DFA_SIZE_LIMIT,
determinize_bytes: SINGLE_DENSE_DETERMINIZE_SIZE_LIMIT,
};
const SINGLE_DENSE_PROMOTION_THRESHOLD: usize = 64;
struct SingleCaches {
seek: hybrid::Cache,
accept: Option<hybrid::Cache>,
}
type CacheMaker = Box<dyn Fn() -> SingleCaches + Send + Sync>;
const SINGLE_CACHE_CAPACITY: usize = MIB;
fn nfa_states_estimate(hir: &Hir) -> usize {
states_estimate(hir, false)
}
fn states_estimate(hir: &Hir, utf8_classes: bool) -> usize {
match hir.kind() {
HirKind::Empty | HirKind::Look(_) => 1,
HirKind::Literal(literal) => literal.0.len() + 1,
HirKind::Class(Class::Unicode(class)) if utf8_classes => {
class.ranges().iter().fold(2usize, |sum, range| {
Utf8Sequences::new(range.start(), range.end())
.fold(sum, |sum, sequence| sum.saturating_add(sequence.len()))
})
}
HirKind::Class(_) => 2,
HirKind::Capture(capture) => states_estimate(&capture.sub, utf8_classes) + 2,
HirKind::Repetition(rep) => {
let copies = rep.max.unwrap_or(rep.min).max(rep.min).max(1) as usize;
states_estimate(&rep.sub, utf8_classes)
.saturating_mul(copies)
.saturating_add(2)
}
HirKind::Concat(parts) => parts.iter().fold(0usize, |sum, part| {
sum.saturating_add(states_estimate(part, utf8_classes))
}),
HirKind::Alternation(branches) => branches.iter().fold(2usize, |sum, branch| {
sum.saturating_add(states_estimate(branch, utf8_classes))
}),
}
}
impl SingleSeek {
pub(crate) fn prepare(&self) {
if let Some(adaptive) = self.adaptive_seek() {
adaptive.initialize_dense_with(|| {
SingleTable::build(self.dfa.get_nfa(), adaptive.limits).map(Box::new)
});
}
if let Some(adaptive) = self.adaptive_accept()
&& let Some((dfa, _)) = &self.accept
{
adaptive.initialize_dense_with(|| {
SingleTable::build(dfa.get_nfa(), adaptive.limits).map(Box::new)
});
}
}
pub(crate) fn build(seek: &Seek, accept: Option<&AcceptCode>) -> Option<Self> {
let seek_limits = match seek {
Seek::Pattern(code) if code.exact() => Some(PREFERRED_SINGLE_DENSE_LIMITS),
_ => None,
};
Self::assemble(
seek,
accept,
seek_limits,
Some(PREFERRED_SINGLE_DENSE_LIMITS),
SINGLE_DENSE_PROMOTION_THRESHOLD,
false,
)
}
#[cfg(test)]
fn build_with_adaptive_policy(
seek: &Seek,
accept: Option<&AcceptCode>,
limits: Option<SingleDenseLimits>,
threshold: usize,
) -> Option<Self> {
Self::assemble(seek, accept, limits, limits, threshold, false)
}
#[cfg(test)]
fn build_with_dense_limits(
seek: &Seek,
accept: Option<&AcceptCode>,
limits: Option<SingleDenseLimits>,
) -> Option<Self> {
Self::build_with_separate_dense_limits(seek, accept, limits, limits)
}
#[cfg(test)]
fn build_with_separate_dense_limits(
seek: &Seek,
accept: Option<&AcceptCode>,
seek_limits: Option<SingleDenseLimits>,
accept_limits: Option<SingleDenseLimits>,
) -> Option<Self> {
Self::assemble(seek, accept, seek_limits, accept_limits, 0, true)
}
fn assemble(
seek: &Seek,
accept: Option<&AcceptCode>,
seek_limits: Option<SingleDenseLimits>,
accept_limits: Option<SingleDenseLimits>,
threshold: usize,
eager: bool,
) -> Option<Self> {
let Seek::Pattern(code) = seek else {
return None;
};
let (dfa, dense) = single_dfa_adaptive(&code.decode(), seek_limits, threshold, eager)?;
let accept = accept.and_then(|accept| {
let (dfa, dense) =
single_dfa_adaptive(&accept.code.decode(), accept_limits, threshold, eager)?;
Some((dfa, accept.rejects.clone(), dense))
});
let (accept, accept_dense) = match accept {
Some((dfa, rejects, dense)) => (Some((dfa, rejects)), dense),
None => (None, None),
};
let make_seek = dfa.clone();
let make_accept = accept.as_ref().map(|(dfa, _)| dfa.clone());
let caches = Pool::new(Box::new(move || SingleCaches {
seek: make_seek.create_cache(),
accept: make_accept.as_ref().map(hybrid::DFA::create_cache),
}) as CacheMaker);
Some(Self {
dfa,
accept,
caches,
exact: code.exact(),
adaptive: (dense.is_some() || accept_dense.is_some()).then(|| {
Box::new(SingleAdaptive {
seek: dense,
accept: accept_dense,
})
}),
})
}
pub(crate) fn search(&self, text: &[u8], start: usize) -> Option<bool> {
let Some(adaptive) = self.adaptive_seek() else {
return self.search_hybrid(text, start);
};
if let Some(table) = adaptive.dense() {
return Some(table.search(text, start));
}
let result = self.search_hybrid(text, start);
adaptive.record_hybrid_consultation(self.dfa.get_nfa());
result
}
pub(crate) fn search_hybrid(&self, text: &[u8], start: usize) -> Option<bool> {
let input = Input::new(text).span(start..text.len()).earliest(true);
let mut caches = self.caches.get();
self.dfa
.try_search_fwd(&mut caches.seek, &input)
.ok()
.map(|found| found.is_some())
}
fn adaptive_seek(&self) -> Option<&AdaptiveDfa> {
self.adaptive.as_deref()?.seek.as_ref()
}
fn adaptive_accept(&self) -> Option<&AdaptiveDfa> {
self.adaptive.as_deref()?.accept.as_ref()
}
pub(crate) fn exact(&self) -> bool {
self.exact
}
pub(crate) fn has_accept(&self) -> bool {
self.accept.is_some()
}
pub(crate) fn accept_answer(&self, text: &[u8], start: usize) -> Option<bool> {
let (dfa, rejects) = self.accept.as_ref()?;
let adaptive = self.adaptive_accept();
let found = if let Some(table) = adaptive.and_then(AdaptiveDfa::dense) {
table.search(text, start)
} else {
let input = Input::new(text).span(start..text.len()).earliest(true);
let result = {
let mut caches = self.caches.get();
dfa.try_search_fwd(caches.accept.as_mut()?, &input)
.ok()
.map(|found| found.is_some())
};
if let Some(adaptive) = adaptive {
adaptive.record_hybrid_consultation(dfa.get_nfa());
}
result?
};
if found {
return Some(true);
}
match rejects {
accept::AcceptRejects::Unless(triggers) => {
(!triggers.occur_in(&text[start..])).then_some(false)
}
accept::AcceptRejects::Never => None,
}
}
#[cfg(test)]
pub(crate) fn memory_usage(&self) -> usize {
self.automata_memory_usage() + self.cache_memory_usage()
}
#[cfg(test)]
fn uses_dense_dfas(&self) -> bool {
self.adaptive_seek().and_then(AdaptiveDfa::dense).is_some()
&& (self.accept.is_none()
|| self
.adaptive_accept()
.and_then(AdaptiveDfa::dense)
.is_some())
}
#[cfg(test)]
pub(crate) fn backend_names(&self) -> (&'static str, Option<&'static str>) {
(
if self.adaptive_seek().and_then(AdaptiveDfa::dense).is_some() {
"dense"
} else {
"hybrid"
},
self.accept.as_ref().map(|_| {
if self
.adaptive_accept()
.and_then(AdaptiveDfa::dense)
.is_some()
{
"dense"
} else {
"hybrid"
}
}),
)
}
#[cfg(test)]
fn automata_memory_usage(&self) -> usize {
fn table_bytes(adaptive: Option<&AdaptiveDfa>) -> usize {
adaptive.and_then(AdaptiveDfa::dense).map_or(0, |table| {
table.memory_usage() + std::mem::size_of_val(table)
})
}
self.dfa.get_nfa().memory_usage()
+ std::mem::size_of_val(&self.dfa)
+ self.accept.as_ref().map_or(0, |(dfa, _)| {
dfa.get_nfa().memory_usage() + std::mem::size_of_val(dfa)
})
+ self.adaptive.as_deref().map_or(0, std::mem::size_of_val)
+ table_bytes(self.adaptive_seek())
+ table_bytes(self.adaptive_accept())
}
#[cfg(test)]
fn cache_memory_usage(&self) -> usize {
let caches = self.caches.get();
caches.seek.memory_usage()
+ caches
.accept
.as_ref()
.map_or(0, hybrid::Cache::memory_usage)
}
}
fn single_nfa(hir: &Hir) -> Option<(thompson::NFA, bool)> {
let estimated_states = nfa_states_estimate(hir);
if estimated_states > MAX_NFA_STATES {
return None;
}
let dense_eligible = estimated_states <= MAX_SINGLE_DENSE_ESTIMATED_NFA_STATES;
let nfa = thompson::Compiler::new()
.configure(
thompson::Config::new()
.utf8(false)
.which_captures(thompson::WhichCaptures::None)
.nfa_size_limit(Some(NFA_SIZE_LIMIT)),
)
.build_from_hir(hir)
.ok()?;
let actual_states = nfa.states().len();
if actual_states > MAX_NFA_STATES {
return None;
}
Some((
nfa,
dense_eligible && actual_states <= MAX_SINGLE_BITSET_NFA_STATES,
))
}
fn build_single_hybrid(nfa: thompson::NFA) -> Option<hybrid::DFA> {
hybrid::Builder::new()
.configure(
hybrid::Config::new()
.match_kind(MatchKind::LeftmostFirst)
.cache_capacity(SINGLE_CACHE_CAPACITY),
)
.build_from_nfa(nfa)
.ok()
}
fn single_dfa_adaptive(
hir: &Hir,
limits: Option<SingleDenseLimits>,
threshold: usize,
eager: bool,
) -> Option<(hybrid::DFA, Option<AdaptiveDfa>)> {
let (nfa, eligible) = single_nfa(hir)?;
let dense = if let Some(limits) = limits
&& eligible
&& SingleTable::supports(&nfa)
{
let adaptive = AdaptiveDfa::new(limits, threshold);
if eager {
adaptive.initialize_dense_with(|| SingleTable::build(&nfa, limits).map(Box::new));
}
Some(adaptive)
} else {
None
};
Some((build_single_hybrid(nfa)?, dense))
}
#[cfg(test)]
mod tests {
use super::*;
use crate::encodings::utf8::ONIG_ENCODING_UTF8;
use crate::oniguruma::{
ONIG_OPTION_CAPTURE_GROUP, ONIG_OPTION_IGNORECASE, ONIG_OPTION_WORD_IS_ASCII,
OnigOptionType,
};
use crate::regcomp::{onig_new_for_scanner, onig_new_for_single_seek};
use crate::regsyntax::OnigSyntaxOniguruma;
use crate::scanner::{Scanner, ScannerConfig, ScannerFindOptions};
const DJANGO_EMAIL_LOCAL: &str = r#"(^[-!#$%&'*+/=?^_`{}|~0-9A-Z]+(\.[-!#$%&'*+/=?^_`{}|~0-9A-Z]+)*\z|^"([\001-\010\013\014\016-\037!#-\[\]-\177]|\\[\001-\011\013\014\016-\177])*"\z)"#;
const DJANGO_DOMAIN: &str = r"^[a-z¡-0-9](?:[a-z¡-0-9-]{0,61}[a-z¡-0-9])?(?:\.(?!-)[a-z¡-0-9-]{1,63}(?<!-))*\.(?!-)(?:[a-z¡--]{2,63}|xn--[a-z0-9]{1,59})(?<!-)\z";
const DJANGO_URL: &str = r"^(?:[a-z0-9.+-]*)://(?:[^\s:@/]+(?::[^\s:@/]*)?@)?(?:(?:0|25[0-5]|2[0-4][0-9]|1[0-9]?[0-9]?|[1-9][0-9]?)(?:\.(?:0|25[0-5]|2[0-4][0-9]|1[0-9]?[0-9]?|[1-9][0-9]?)){3}|\[[0-9a-f:.]+\]|([a-z¡-0-9](?:[a-z¡-0-9-]{0,61}[a-z¡-0-9])?(?:\.(?!-)[a-z¡-0-9-]{1,63}(?<!-))*\.(?!-)(?:[a-z¡--]{2,63}|xn--[a-z0-9]{1,59})(?<!-)\.?|localhost))(?::[0-9]{1,5})?(?:[/?#][^\s]*)?\z";
const DJANGO_OPTIONS: OnigOptionType = ONIG_OPTION_CAPTURE_GROUP
.union(ONIG_OPTION_IGNORECASE)
.union(crate::oniguruma::ONIG_OPTION_SINGLELINE);
fn seek_with(
pattern: &str,
options: OnigOptionType,
) -> Result<Option<Seek>, crate::error::RegexError> {
onig_new_for_scanner(
pattern.as_bytes(),
options,
&ONIG_ENCODING_UTF8,
&OnigSyntaxOniguruma,
false,
true,
)
.map(|(_, seek)| seek)
}
fn seek(pattern: &str) -> Option<Seek> {
seek_with(pattern, ONIG_OPTION_CAPTURE_GROUP).unwrap()
}
fn single_seek_with(
pattern: &str,
options: OnigOptionType,
dense_limits: Option<SingleDenseLimits>,
) -> Option<SingleSeek> {
let (_, seek, accept) = onig_new_for_single_seek(
pattern.as_bytes(),
options,
&ONIG_ENCODING_UTF8,
&OnigSyntaxOniguruma,
false,
true,
)
.unwrap();
SingleSeek::build_with_dense_limits(&seek?, accept.as_ref(), dense_limits)
}
fn adaptive_single_seek_with(
pattern: &str,
options: OnigOptionType,
dense_limits: Option<SingleDenseLimits>,
threshold: usize,
) -> Option<SingleSeek> {
let (_, seek, accept) = onig_new_for_single_seek(
pattern.as_bytes(),
options,
&ONIG_ENCODING_UTF8,
&OnigSyntaxOniguruma,
false,
true,
)
.unwrap();
SingleSeek::build_with_adaptive_policy(&seek?, accept.as_ref(), dense_limits, threshold)
}
fn production_single_seek(pattern: &str, options: OnigOptionType) -> Option<SingleSeek> {
let (_, seek, accept) = onig_new_for_single_seek(
pattern.as_bytes(),
options,
&ONIG_ENCODING_UTF8,
&OnigSyntaxOniguruma,
false,
true,
)
.unwrap();
SingleSeek::build(&seek?, accept.as_ref())
}
fn char_boundaries(text: &str) -> Vec<usize> {
text.char_indices()
.map(|(index, _)| index)
.chain(std::iter::once(text.len()))
.collect()
}
impl SingleSeek {
fn adaptive_consultations(&self) -> (Option<usize>, Option<usize>) {
(
self.adaptive_seek().map(AdaptiveDfa::consultation_count),
self.adaptive_accept().map(AdaptiveDfa::consultation_count),
)
}
}
#[test]
fn django_catalog_nfa_size_diagnostic() {
eprintln!(
"small dense eligibility: estimated NFA states <= {}, actual <=64; final DFA <= {} B; determinization scratch <= {} B",
MAX_SINGLE_DENSE_ESTIMATED_NFA_STATES,
SINGLE_DENSE_DFA_SIZE_LIMIT,
SINGLE_DENSE_DETERMINIZE_SIZE_LIMIT,
);
let local_options = DJANGO_OPTIONS;
for (name, pattern, options, expected_eligible) in [
(
"django_email_local_part",
DJANGO_EMAIL_LOCAL,
local_options,
true,
),
("django_email_domain", DJANGO_DOMAIN, DJANGO_OPTIONS, false),
("django_url_regex", DJANGO_URL, DJANGO_OPTIONS, false),
] {
let (_, seek, accept) = onig_new_for_single_seek(
pattern.as_bytes(),
options,
&ONIG_ENCODING_UTF8,
&OnigSyntaxOniguruma,
false,
true,
)
.unwrap();
let Seek::Pattern(seek) = seek.expect("catalog pattern has a seek") else {
panic!("{name} seek is not a pattern");
};
let mut hirs = vec![("seek", seek.decode())];
if let Some(accept) = accept.as_ref() {
hirs.push(("accept", accept.code.decode()));
}
if name == "django_email_local_part" {
assert_eq!(hirs.len(), 1, "local-part seek is exact");
}
let mut seek_eligible = false;
for (kind, hir) in hirs {
let gate_estimate = nfa_states_estimate(&hir);
let estimated = if kind == "accept" {
states_estimate(&hir, true)
} else {
gate_estimate
};
let (nfa, eligible) = single_nfa(&hir).expect("catalog NFA builds");
let actual = nfa.states().len();
assert_eq!(
eligible,
gate_estimate <= MAX_SINGLE_DENSE_ESTIMATED_NFA_STATES
&& actual <= MAX_SINGLE_BITSET_NFA_STATES
);
assert_eq!(
eligible,
kind == "seek" && expected_eligible,
"{name} {kind} eligibility under the small policy"
);
eprintln!(
"{name} {kind}: estimated_states={estimated}, dense_gate_estimate={gate_estimate}, actual_states={actual}, dense_eligible={eligible}"
);
if kind == "seek" {
seek_eligible = eligible;
}
}
assert_eq!(seek_eligible, expected_eligible, "{name} selection");
}
}
#[test]
fn dense_single_seeks_match_the_hybrid_fallback() {
let cases = [
(
r"\A(?:ab|cd)[0-9]{1,3}\z",
ONIG_OPTION_CAPTURE_GROUP,
&["ab7", "x ab7", "cd123", "ab1234", "zz", "abc"] as &[&str],
),
(
r"(?i)k+\z",
ONIG_OPTION_CAPTURE_GROUP | ONIG_OPTION_IGNORECASE,
&["k", "Kk", "\u{212A}k", "xk", "k "] as &[&str],
),
(
r"(?m)^あ+$",
ONIG_OPTION_CAPTURE_GROUP,
&["あ", "x\nあ\n", "あい", "xあ"] as &[&str],
),
(
r"[é-ü]+x(?i:[ß])?",
ONIG_OPTION_CAPTURE_GROUP,
&["éx", "üx", "éxß", "éxss", "x", "éy"] as &[&str],
),
];
let dense_limits = Some(PREFERRED_SINGLE_DENSE_LIMITS);
for (pattern, options, subjects) in cases {
let dense = single_seek_with(pattern, options, dense_limits)
.unwrap_or_else(|| panic!("no single seek for {pattern:?}"));
let hybrid =
single_seek_with(pattern, options, None).expect("the hybrid fallback builds");
for text in subjects {
for start in char_boundaries(text) {
assert_eq!(
dense.search(text.as_bytes(), start),
hybrid.search(text.as_bytes(), start),
"seek for {pattern:?} on {text:?} from {start}"
);
assert_eq!(
dense.accept_answer(text.as_bytes(), start),
hybrid.accept_answer(text.as_bytes(), start),
"accept seek for {pattern:?} on {text:?} from {start}"
);
}
}
}
}
#[test]
fn dense_single_seek_is_bounded_and_falls_back_to_hybrid() {
let pattern = r"\A(?:ab|cd)[0-9]{1,3}\z";
let dense = single_seek_with(
pattern,
ONIG_OPTION_CAPTURE_GROUP,
Some(SingleDenseLimits {
dfa_bytes: SINGLE_DENSE_DFA_SIZE_LIMIT,
determinize_bytes: SINGLE_DENSE_DETERMINIZE_SIZE_LIMIT,
}),
)
.expect("seek builds");
assert!(dense.uses_dense_dfas());
assert!(dense.memory_usage() <= SINGLE_DENSE_DFA_SIZE_LIMIT);
let forced = single_seek_with(
pattern,
ONIG_OPTION_CAPTURE_GROUP,
Some(SingleDenseLimits {
dfa_bytes: 0,
determinize_bytes: 0,
}),
)
.expect("hybrid fallback builds after dense rejection");
assert_eq!(forced.backend_names().0, "hybrid");
for text in ["ab1", "cd123", "ab1234", "zz"] {
assert_eq!(
dense.search(text.as_bytes(), 0),
forced.search(text.as_bytes(), 0)
);
}
}
#[test]
fn adaptive_single_seek_promotes_each_dfa_after_its_consultation_threshold() {
let pattern = r"[é-ü]+x(?i:[ß])?";
let adaptive = adaptive_single_seek_with(
pattern,
ONIG_OPTION_CAPTURE_GROUP,
Some(PREFERRED_SINGLE_DENSE_LIMITS),
2,
)
.expect("adaptive seek builds");
let hybrid = single_seek_with(pattern, ONIG_OPTION_CAPTURE_GROUP, None)
.expect("forced-hybrid reference builds");
assert_eq!(adaptive.backend_names(), ("hybrid", Some("hybrid")));
assert_eq!(adaptive.adaptive_consultations(), (Some(0), Some(0)));
assert_eq!(
adaptive.search("éx".as_bytes(), 0),
hybrid.search("éx".as_bytes(), 0)
);
assert_eq!(
adaptive.accept_answer("éx".as_bytes(), 0),
hybrid.accept_answer("éx".as_bytes(), 0)
);
assert_eq!(adaptive.adaptive_consultations(), (Some(1), Some(1)));
assert_eq!(adaptive.backend_names(), ("hybrid", Some("hybrid")));
assert_eq!(
adaptive.search("éxß".as_bytes(), 0),
hybrid.search("éxß".as_bytes(), 0)
);
assert_eq!(adaptive.backend_names(), ("dense", Some("hybrid")));
assert_eq!(adaptive.adaptive_consultations(), (Some(2), Some(1)));
assert_eq!(
adaptive.accept_answer("éxß".as_bytes(), 0),
hybrid.accept_answer("éxß".as_bytes(), 0)
);
assert_eq!(adaptive.backend_names(), ("dense", Some("dense")));
assert_eq!(adaptive.adaptive_consultations(), (Some(2), Some(2)));
for text in ["éx", "éxß", "x", "éy"] {
assert_eq!(
adaptive.search(text.as_bytes(), 0),
hybrid.search(text.as_bytes(), 0)
);
assert_eq!(
adaptive.accept_answer(text.as_bytes(), 0),
hybrid.accept_answer(text.as_bytes(), 0)
);
}
assert_eq!(adaptive.adaptive_consultations(), (Some(2), Some(2)));
}
#[test]
fn adaptive_single_seek_memoizes_dense_rejection_and_keeps_hybrid() {
let pattern = r"(?:ab|cd)[0-9]{1,3}";
let adaptive = adaptive_single_seek_with(
pattern,
ONIG_OPTION_CAPTURE_GROUP,
Some(SingleDenseLimits {
dfa_bytes: 0,
determinize_bytes: 0,
}),
2,
)
.expect("adaptive seek builds");
let hybrid = single_seek_with(pattern, ONIG_OPTION_CAPTURE_GROUP, None)
.expect("forced-hybrid reference builds");
for _ in 0..2 {
assert_eq!(
adaptive.search("cd123".as_bytes(), 0),
hybrid.search("cd123".as_bytes(), 0)
);
}
let adaptive_dfa = adaptive.adaptive_seek().expect("seek is adaptive");
assert!(adaptive_dfa.promotion_attempted());
assert!(matches!(adaptive_dfa.dense_decision.get(), Some(None)));
assert_eq!(adaptive.adaptive_consultations().0, Some(2));
assert_eq!(adaptive.backend_names().0, "hybrid");
for text in ["ab7", "cd123", "cd1234", "none"] {
assert_eq!(
adaptive.search(text.as_bytes(), 0),
hybrid.search(text.as_bytes(), 0)
);
}
assert_eq!(adaptive.adaptive_consultations().0, Some(2));
assert!(matches!(adaptive_dfa.dense_decision.get(), Some(None)));
}
#[test]
fn adaptive_single_seek_promotion_panic_falls_back_without_retry() {
let pattern = r"[é-ü]+x(?i:[ß])?";
let adaptive = adaptive_single_seek_with(
pattern,
ONIG_OPTION_CAPTURE_GROUP,
Some(PREFERRED_SINGLE_DENSE_LIMITS),
1,
)
.expect("adaptive seek builds");
let hybrid = single_seek_with(pattern, ONIG_OPTION_CAPTURE_GROUP, None)
.expect("forced-hybrid reference builds");
let adaptive_dfa = adaptive.adaptive_seek().expect("seek is adaptive");
let attempts = AtomicUsize::new(0);
let panic = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
adaptive_dfa.initialize_dense_with(|| {
attempts.fetch_add(1, Ordering::Relaxed);
panic!("injected dense builder panic");
});
}));
assert!(panic.is_err());
assert_eq!(attempts.load(Ordering::Relaxed), 1);
assert!(adaptive_dfa.promotion_attempted());
assert!(adaptive_dfa.dense_decision.get().is_none());
adaptive_dfa.initialize_dense_with(|| {
attempts.fetch_add(1, Ordering::Relaxed);
None
});
assert_eq!(attempts.load(Ordering::Relaxed), 1);
assert_eq!(
adaptive.search("éxß".as_bytes(), 0),
hybrid.search("éxß".as_bytes(), 0)
);
assert_eq!(adaptive.backend_names().0, "hybrid");
assert_eq!(adaptive.adaptive_consultations().0, Some(0));
}
#[test]
fn adaptive_single_seek_promotes_safely_across_threads() {
let pattern = r"[é-ü]+x(?i:[ß])?";
let adaptive = Arc::new(
adaptive_single_seek_with(
pattern,
ONIG_OPTION_CAPTURE_GROUP,
Some(PREFERRED_SINGLE_DENSE_LIMITS),
64,
)
.expect("adaptive seek builds"),
);
let hybrid = Arc::new(
single_seek_with(pattern, ONIG_OPTION_CAPTURE_GROUP, None)
.expect("forced-hybrid reference builds"),
);
let mut workers = Vec::new();
for _ in 0..4 {
let adaptive = Arc::clone(&adaptive);
let hybrid = Arc::clone(&hybrid);
workers.push(std::thread::spawn(move || {
for iteration in 0..24 {
let text = if iteration % 2 == 0 { "éx" } else { "éxß" };
assert_eq!(
adaptive.search(text.as_bytes(), 0),
hybrid.search(text.as_bytes(), 0)
);
assert_eq!(
adaptive.accept_answer(text.as_bytes(), 0),
hybrid.accept_answer(text.as_bytes(), 0)
);
}
}));
}
for worker in workers {
worker.join().expect("worker completed without panic");
}
assert_eq!(adaptive.backend_names(), ("dense", Some("dense")));
assert_eq!(adaptive.adaptive_consultations(), (Some(64), Some(64)));
}
#[test]
fn django_catalog_single_seek_backend_diagnostic() {
eprintln!(
concat!(
"single-dense eligibility: estimated_states<={}, actual_states<=64; ",
"dfa_bytes<={}, ",
"determinize_scratch_bytes<={}; ",
"hybrid fallback retained if any dense condition fails"
),
MAX_SINGLE_DENSE_ESTIMATED_NFA_STATES,
SINGLE_DENSE_DFA_SIZE_LIMIT,
SINGLE_DENSE_DETERMINIZE_SIZE_LIMIT,
);
let mut dense_seek = false;
let mut dense_accept = false;
for (name, pattern, subjects) in [
(
"django_email_domain",
DJANGO_DOMAIN,
&[
"example.com",
"straße.de",
"strasse.de",
"a-.com",
"localhost",
] as &[&str],
),
(
"django_url_regex",
DJANGO_URL,
&[
"http://example.com/",
"http://straße.de/",
"http://strasse.de/",
"http://-a.com/",
"http://localhost/",
"not a url",
] as &[&str],
),
] {
let seek =
single_seek_with(pattern, DJANGO_OPTIONS, Some(PREFERRED_SINGLE_DENSE_LIMITS))
.unwrap_or_else(|| panic!("no seek for {name}"));
let hybrid = single_seek_with(pattern, DJANGO_OPTIONS, None)
.unwrap_or_else(|| panic!("no hybrid seek for {name}"));
let (seek_backend, accept_backend) = seek.backend_names();
dense_seek |= seek_backend == "dense";
dense_accept |= accept_backend == Some("dense");
assert!(seek.has_accept(), "{name} keeps its accept seek");
assert_eq!(
seek_backend, "hybrid",
"{name} exceeds the small-table guards"
);
assert_eq!(
accept_backend,
Some("hybrid"),
"{name} accept exceeds the gate"
);
assert!(
seek.automata_memory_usage() <= 2 * SINGLE_DENSE_DFA_SIZE_LIMIT,
"{name} diagnostic exceeds the combined small-table cap"
);
let mut comparisons = 0;
for text in subjects {
for start in char_boundaries(text) {
comparisons += 1;
assert_eq!(
seek.search(text.as_bytes(), start),
hybrid.search(text.as_bytes(), start),
"{name} seek on {text:?} from {start}"
);
assert_eq!(
seek.accept_answer(text.as_bytes(), start),
hybrid.accept_answer(text.as_bytes(), start),
"{name} accept seek on {text:?} from {start}"
);
}
}
eprintln!(
concat!(
"{}: seek={}, accept={:?}; ",
"configured automata_only={} B, warmed_current_thread_caches={} B; ",
"forced_hybrid automata_only={} B, warmed_current_thread_caches={} B; ",
"each backend ran search and accept_answer at {} ",
"character-boundary positions; cache sizes are after those calls ",
"(pool bookkeeping excluded)"
),
name,
seek_backend,
accept_backend,
seek.automata_memory_usage(),
seek.cache_memory_usage(),
hybrid.automata_memory_usage(),
hybrid.cache_memory_usage(),
comparisons,
);
}
assert!(
!dense_seek,
"large catalog seeks stay hybrid under the small gate"
);
assert!(
!dense_accept,
"large catalog accepts stay hybrid under the small gate"
);
}
#[test]
fn django_catalog_adaptive_single_seek_diagnostic() {
eprintln!(
concat!(
"production adaptive policy: estimated_states<={}, actual<=64, text anchors only, promotion_after={} actual DFA consultations, ",
"dense_bytes<={}, determinize_scratch_bytes<={}; ",
"hybrid NFA/backend and per-thread caches remain allocated after promotion"
),
MAX_SINGLE_DENSE_ESTIMATED_NFA_STATES,
SINGLE_DENSE_PROMOTION_THRESHOLD,
SINGLE_DENSE_DFA_SIZE_LIMIT,
SINGLE_DENSE_DETERMINIZE_SIZE_LIMIT,
);
let mut dense_seek = false;
let mut dense_accept = false;
let mut local_part_promoted = false;
let local_options = DJANGO_OPTIONS;
for (name, pattern, options, text, eligible) in [
(
"django_email_local_part",
DJANGO_EMAIL_LOCAL,
local_options,
"john.doe",
true,
),
(
"django_email_domain",
DJANGO_DOMAIN,
DJANGO_OPTIONS,
"example.com",
false,
),
(
"django_url_regex",
DJANGO_URL,
DJANGO_OPTIONS,
"http://example.com/",
false,
),
] {
let adaptive = production_single_seek(pattern, options)
.unwrap_or_else(|| panic!("no production seek for {name}"));
let initial_backends = adaptive.backend_names();
assert_eq!(initial_backends.0, "hybrid");
let initial_automata = adaptive.automata_memory_usage();
if eligible {
assert_eq!(adaptive.adaptive_consultations(), (Some(0), None));
assert_eq!(initial_backends.1, None, "local-part seek is exact");
for _ in 0..(SINGLE_DENSE_PROMOTION_THRESHOLD - 1) {
let _ = adaptive.search(text.as_bytes(), 0);
}
assert_eq!(adaptive.backend_names(), ("hybrid", None));
let before_automata = adaptive.automata_memory_usage();
let before_caches = adaptive.cache_memory_usage();
let before_counts = adaptive.adaptive_consultations();
assert_eq!(before_counts, (Some(63), None));
let _ = adaptive.search(text.as_bytes(), 0);
let after_backends = adaptive.backend_names();
let after_automata = adaptive.automata_memory_usage();
let after_caches = adaptive.cache_memory_usage();
let after_counts = adaptive.adaptive_consultations();
local_part_promoted = after_backends.0 == "dense";
dense_seek |= local_part_promoted;
dense_accept |= after_backends.1 == Some("dense");
eprintln!(
concat!(
"{} eligible: initial_automata={} B; before_promotion={:?}, ",
"automata={} B, warmed_current_thread_caches={} B; ",
"after={:?}, automata={} B, retained_current_thread_caches={} B; counts={:?}"
),
name,
initial_automata,
before_counts,
before_automata,
before_caches,
after_backends,
after_automata,
after_caches,
after_counts,
);
assert_eq!(after_counts, (Some(64), None));
assert!(after_automata >= before_automata);
} else {
assert_eq!(adaptive.adaptive_consultations(), (None, None));
for _ in 0..SINGLE_DENSE_PROMOTION_THRESHOLD {
let _ = adaptive.search(text.as_bytes(), 0);
let _ = adaptive.accept_answer(text.as_bytes(), 0);
}
let after_backends = adaptive.backend_names();
let after_automata = adaptive.automata_memory_usage();
let after_caches = adaptive.cache_memory_usage();
eprintln!(
concat!(
"{} ineligible: initial_automata={} B; after_64_search_and_accept={:?}, ",
"automata={} B, warmed_current_thread_caches={} B; counts={:?}"
),
name,
initial_automata,
after_backends,
after_automata,
after_caches,
adaptive.adaptive_consultations(),
);
assert_eq!(after_backends, ("hybrid", Some("hybrid")));
assert_eq!(adaptive.adaptive_consultations(), (None, None));
assert_eq!(after_automata, initial_automata);
}
}
assert!(
local_part_promoted && dense_seek,
"production policy promotes the eligible local-part seek"
);
assert!(
!dense_accept,
"catalog accepts exceed the small-table guards"
);
}
#[test]
fn mixed_dense_and_hybrid_seek_uses_only_hybrid_caches() {
let pattern = r"[é-ü]+x(?i:[ß])?";
let (_, seek, accept) = onig_new_for_single_seek(
pattern.as_bytes(),
ONIG_OPTION_CAPTURE_GROUP,
&ONIG_ENCODING_UTF8,
&OnigSyntaxOniguruma,
false,
true,
)
.unwrap();
let seek = seek.unwrap();
let hybrid = SingleSeek::build_with_dense_limits(&seek, accept.as_ref(), None)
.expect("hybrid pair builds");
let mixed = SingleSeek::build_with_separate_dense_limits(
&seek,
accept.as_ref(),
Some(PREFERRED_SINGLE_DENSE_LIMITS),
Some(SingleDenseLimits {
dfa_bytes: 0,
determinize_bytes: 0,
}),
)
.expect("mixed pair builds");
assert_eq!(mixed.backend_names(), ("dense", Some("hybrid")));
for text in ["éx", "éxß", "éxss", "x", "éy"] {
assert_eq!(
mixed.search(text.as_bytes(), 0),
hybrid.search(text.as_bytes(), 0)
);
assert_eq!(
mixed.accept_answer(text.as_bytes(), 0),
hybrid.accept_answer(text.as_bytes(), 0)
);
}
let reverse_mixed = SingleSeek::build_with_separate_dense_limits(
&seek,
accept.as_ref(),
Some(SingleDenseLimits {
dfa_bytes: 0,
determinize_bytes: 0,
}),
Some(PREFERRED_SINGLE_DENSE_LIMITS),
)
.expect("reverse mixed pair builds");
assert_eq!(reverse_mixed.backend_names(), ("hybrid", Some("dense")));
for text in ["éx", "éxß", "éxss", "x", "éy"] {
assert_eq!(
reverse_mixed.search(text.as_bytes(), 0),
hybrid.search(text.as_bytes(), 0)
);
assert_eq!(
reverse_mixed.accept_answer(text.as_bytes(), 0),
hybrid.accept_answer(text.as_bytes(), 0)
);
}
}
#[test]
fn dense_single_seek_stays_send_and_sync() {
fn assert_send_sync<T: Send + Sync>() {}
assert_send_sync::<SingleSeek>();
}
fn seek_pattern_with(pattern: &str, options: OnigOptionType) -> String {
match seek_with(pattern, options) {
Ok(Some(Seek::Pattern(seek))) => seek.decode().to_string(),
other => format!("{other:?}"),
}
}
fn seek_pattern(pattern: &str) -> String {
seek_pattern_with(pattern, ONIG_OPTION_CAPTURE_GROUP)
}
fn exact_with(pattern: &str, options: OnigOptionType) -> Option<bool> {
let seek = crate::regcomp::onig_new_for_single_seek(
pattern.as_bytes(),
options,
&ONIG_ENCODING_UTF8,
&OnigSyntaxOniguruma,
false,
true,
)
.unwrap()
.1;
match seek {
Some(Seek::Pattern(code)) => Some(code.exact()),
_ => None,
}
}
#[test]
fn seeks_are_exact_exactly_where_the_matcher_reads_alike() {
let exact = [
"abc",
"[a-c]",
"a|bc",
"ab{2,5}c",
"[^a]",
".",
"\u{3042}+",
"\\x{1F600}",
"x|$",
"a$",
"\\Aab\\z",
"a\\Kb",
"^a",
"^[0-9]{8}$",
"(?:^a|^b)c",
"(?i)[0-9a-f]+",
"(?i)k",
"(?i)s",
"[\u{e9}]",
"(a)(b)",
"^[a-z]+@[a-z]+\\.com$",
"[a-z]*@",
"[^\"]*\"",
"(?:ab)*c",
"([a-z]+)@([a-z]+)",
];
let inexact = [
"(?>abc|ab)c",
"(?>a+)b",
"a++b",
"a*+b",
"[a-z]++@",
"^[a-z]+@(?>[a-z]+)\\.com$",
"(?>[a-z]+)@",
"\\b",
"\\B",
"(?=a)b",
"(?<=a)b",
"a(?!b)",
"a(?<!b)",
"\\Zb",
"^",
"^$",
"(a)\\1",
"\\W",
"\\w",
"\\d+",
"[^\u{e9}]",
"\\G\\w",
"(?:ab|cd){100}",
"(x)?(?(1)a|b)",
];
for pattern in exact {
assert_eq!(
exact_with(pattern, ONIG_OPTION_CAPTURE_GROUP),
Some(true),
"{pattern}"
);
}
for pattern in inexact {
assert_eq!(
exact_with(pattern, ONIG_OPTION_CAPTURE_GROUP),
Some(false),
"{pattern}"
);
}
let ascii_word = ONIG_OPTION_CAPTURE_GROUP | crate::oniguruma::ONIG_OPTION_WORD_IS_ASCII;
assert_eq!(exact_with("\\w+:", ascii_word), Some(true));
assert_eq!(exact_with("\\w+:", ONIG_OPTION_CAPTURE_GROUP), Some(false));
let singleline = ONIG_OPTION_CAPTURE_GROUP | crate::oniguruma::ONIG_OPTION_SINGLELINE;
assert_eq!(exact_with("^a$", singleline), Some(true));
assert_eq!(exact_with("a$|b$", singleline), Some(true));
assert_eq!(exact_with("(?:a|b)\\Z", singleline), Some(true));
assert_eq!(exact_with("a$b", singleline), Some(false));
assert_eq!(exact_with("a$", ONIG_OPTION_CAPTURE_GROUP), Some(true));
assert_eq!(exact_with("(?~abc)x", ONIG_OPTION_CAPTURE_GROUP), None);
assert_eq!(exact_with("(*FAIL)|a", ONIG_OPTION_CAPTURE_GROUP), None);
assert_eq!(exact_with("^a", ONIG_OPTION_NOTBOL), Some(false));
assert_eq!(exact_with("a$", ONIG_OPTION_NOTEOL), Some(false));
assert_eq!(exact_with("a", ONIG_OPTION_FIND_NOT_EMPTY), Some(false));
}
#[test]
fn seeks_approximate_each_construct() {
let cases: &[(&str, &str)] = &[
("abc", "(?:abc)"),
("[a-c]", "(?-u:[a-c])"),
("a|bc", "(?:a|(?:bc))"),
("ab{2,5}c", "(?:ab{2,5}c)"),
("(?:ab)+?", "(?:ab)+"),
("(a)(?:b)(?>c)d?", "(?:(?:abc)d?)"),
("(?:a|bc){70}", "(?:a|(?:bc)){64,}"),
("(?:(?:ab)+)?c", "(?:(?:ab)*c)"),
("(?:(?:ab){2,3}){2}", "(?:ab){4,6}"),
("(?:a*)+b", "(?:a*b)"),
(
"[^x]{2,70}",
"(?:(?-u:[\\x00-wy-\\x7F])|(?:(?-u:[\\xC2-\\xF4])(?-u:[\\x80-\\xBF]){1,3})){2,}",
),
("[é]", "(?:(?-u:[\\xC2-\\xF4])(?-u:[\\x80-\\xBF]){1,3})"),
(
"[^a]",
"(?:(?-u:[\\x00-`b-\\x7F])|(?:(?-u:[\\xC2-\\xF4])(?-u:[\\x80-\\xBF]){1,3}))",
),
(
"\\w",
"(?:(?-u:[0-9A-Z_a-z])|(?:(?-u:[\\xC2-\\xF4])(?-u:[\\x80-\\xBF]){1,3}))",
),
(
".",
"(?:(?-u:[\\x00-\\x09\\x0B-\\x7F])|(?:(?-u:[\\xC2-\\xF4])(?-u:[\\x80-\\xBF]){1,3}))",
),
(
"(?m:.)",
"(?:(?-u:[\\x00-\\x7F])|(?:(?-u:[\\xC2-\\xF4])(?-u:[\\x80-\\xBF]){1,3}))",
),
("\\Aa", "(?:\\Aa)"),
("a\\z", "(?:a\\z)"),
("a\\Z", "(?:a(?m:$))"),
("^a$", "(?:(?m:^)a(?m:$))"),
(
"\\ba",
"(?:(?:(?-u:\\b)|(?-u:\\b{start-half})|(?-u:\\b{end-half}))a)",
),
("(?<=x)a", "a"),
("(?<!x)a", "a"),
("(?<!\\w)a", "(?:(?-u:\\b{start-half})a)"),
(
"(?<![-\\w])a(?![-\\w])",
"(?:(?-u:\\b{start-half})a(?-u:\\b{end-half}))",
),
("(?<!\\W)a", "a"),
("(?<![^\\w])a", "a"),
("(?=ab)(?<!\\w)", "(?-u:\\b{start-half})"),
("(?!x)a", "a"),
("\\Ga", "a"),
("a\\Kb", "(?:ab)"),
("\\ya", "a"),
(
"\\s+(?=use\\b)",
"(?:(?:(?-u:[\\x09-\\x0D\\x20])|(?:(?-u:[\\xC2-\\xF4])(?-u:[\\x80-\\xBF]){1,3}))+(?:use)(?:(?-u:\\b)|(?-u:\\b{start-half})|(?-u:\\b{end-half})))",
),
("(?=a)(?<=b)", "a"),
("(?=a)b", "b"),
("(?=a)(?=b)", "a"),
("y(?:x(?=a))?", "(?:y(?:xa)?)"),
("y(?:x(?=a))+", "(?:yx+)"),
("(?:x(?=a))?y", "(?:x?y)"),
(
"(a)\\1",
"(?:a(?:(?-u:[\\x00-\\x7F])|(?:(?-u:[\\xC2-\\xF4])(?-u:[\\x80-\\xBF]){1,3}))*)",
),
(
"a\\g<1>|(b)",
"(?:(?:a(?:(?-u:[\\x00-\\x7F])|(?:(?-u:[\\xC2-\\xF4])(?-u:[\\x80-\\xBF]){1,3}))*)|b)",
),
("(a)?(?(1)b|c)", "(?:a?[bc])"),
("(?(a)b|c)", "(?:(?:ab)|c)"),
(
"[^[^[^İ]\\S]]",
"(?:(?-u:[\\x00-\\x7F])|(?:(?-u:[\\xC2-\\xF4])(?-u:[\\x80-\\xBF]){1,3}))",
),
("[^[^İ]\\S]", "[a&&b]"),
(
"\\W",
"(?:(?-u:[\\x00-/:-@\\[-\\^`\\{-\\x7F])|(?:(?-u:[\\xC2-\\xF4])(?-u:[\\x80-\\xBF]){1,3}))",
),
("(?i)ab", "(?:(?-u:[Aa])(?-u:[Bb]))"),
(
"(?i)[k]",
"(?:(?-u:[Kk])|(?:(?-u:[\\xC2-\\xF4])(?-u:[\\x80-\\xBF]){1,3}))",
),
(
"(?:error|warn|fatal|panic)",
"(?:(?:error)|(?:warn)|(?:fatal)|(?:panic))",
),
(
"(?i)(?:ab|cd|ef|gh)",
"(?:(?:(?-u:[Aa])(?-u:[Bb]))|(?:(?-u:[Cc])(?-u:[Dd]))|(?:(?-u:[Ee])(?-u:[Ff]))|(?:(?-u:[Gg])(?-u:[Hh]))|(?:(?-u:[\\x00-\\x7F])?[KSks\u{17f}\u{212a}](?:(?-u:[\\x00-\\x7F])|(?:(?-u:[\\xC2-\\xF4])(?-u:[\\x80-\\xBF]){1,3}))?))",
),
(
"(?i)(?:kb|ss|st|xy)",
"(?:(?:(?-u:[Kk])(?-u:[Bb]))|(?:(?-u:[Ss])(?-u:[Ss]))|(?:(?-u:[Ss])(?-u:[Tt]))|(?:(?-u:[Xx])(?-u:[Yy]))|(?:(?-u:[\\x00-\\x7F])?[KSks\u{df}\u{17f}\u{1e9e}\u{212a}\u{fb05}\u{fb06}\u{1df95}](?:(?-u:[\\x00-\\x7F])|(?:(?-u:[\\xC2-\\xF4])(?-u:[\\x80-\\xBF]){1,3}))?))",
),
];
let mismatches: Vec<String> = cases
.iter()
.filter(|(pattern, expected)| seek_pattern(pattern) != *expected)
.map(|(pattern, _)| format!("{pattern:?} => {:?}", seek_pattern(pattern)))
.collect();
assert!(mismatches.is_empty(), "{}", mismatches.join("\n"));
let ascii_word = ONIG_OPTION_CAPTURE_GROUP | ONIG_OPTION_WORD_IS_ASCII;
assert_eq!(seek_pattern_with(r"\w", ascii_word), "(?-u:[0-9A-Z_a-z])");
assert_eq!(seek_pattern_with(r"\ba", ascii_word), "(?:(?-u:\\b)a)");
}
#[test]
fn seeks_that_match_everywhere_leave_the_entry_on_its_own() {
for pattern in [
"",
r"(?<=\))",
r"(?!\s*\[)",
"a*",
r"(a)?\1",
r"\G ?",
r"(?<=}|%>)\s*",
r"(?i)x*",
r"(?~abc)d",
] {
assert_eq!(seek(pattern), Some(Seek::Everywhere), "{pattern}");
}
assert_eq!(seek_pattern("^"), "(?m:^)");
assert_eq!(
seek_pattern(r"\b"),
"(?:(?-u:\\b)|(?-u:\\b{start-half})|(?-u:\\b{end-half}))"
);
}
#[test]
fn seek_codes_round_trip_over_the_captured_grammars() {
let mut patterns = std::collections::BTreeSet::new();
for name in ["cpp", "java", "scss", "c", "php"] {
let path = format!(
"{}/benches/{name}_scanner/trace.json",
env!("CARGO_MANIFEST_DIR")
);
let trace: serde_json::Value =
serde_json::from_str(&std::fs::read_to_string(path).unwrap()).unwrap();
for set in trace["scanners"].as_array().unwrap() {
for pattern in set.as_array().unwrap() {
patterns.insert(pattern.as_str().unwrap().to_owned());
}
}
}
let mut codes = 0usize;
let mut bytes = 0usize;
for pattern in &patterns {
let Some(Seek::Pattern(code)) = seek(pattern) else {
continue;
};
let hir = code.decode();
let again = SeekCode::encode(&hir, code.exact()).expect("the walk's nodes encode");
assert_eq!(again, code, "{pattern}");
assert_eq!(again.decode(), hir, "{pattern}");
codes += 1;
bytes += code.len();
}
assert!(
codes > 600,
"{codes} seeks over {} patterns",
patterns.len()
);
assert!(bytes / codes < 1024, "{bytes} bytes over {codes} seeks");
}
fn flat_grammar_patterns(json: &str) -> Vec<String> {
fn collect(value: &serde_json::Value, out: &mut Vec<String>) {
match value {
serde_json::Value::Object(map) => {
for key in ["match", "begin"] {
if let Some(pattern) = map.get(key).and_then(|v| v.as_str()) {
if !out.iter().any(|p| p == pattern) {
out.push(pattern.to_owned());
}
}
}
for (_, value) in map {
collect(value, out);
}
}
serde_json::Value::Array(items) => {
for item in items {
collect(item, out);
}
}
_ => {}
}
}
let root: serde_json::Value = serde_json::from_str(json).unwrap();
let mut patterns = Vec::new();
collect(&root, &mut patterns);
patterns
.into_iter()
.filter(|pattern| Scanner::new(&[pattern.as_str()]).is_ok())
.collect()
}
fn tokenize(
scanner: &mut Scanner,
line: &crate::scanner::OnigString,
) -> Vec<(usize, usize, usize)> {
let len = line.utf16_len();
let mut pos = 0;
let mut tokens = Vec::new();
while pos < len {
let Some(m) = scanner.find_next_match_utf16(line, pos, ScannerFindOptions::NONE) else {
break;
};
let end = m.captures()[0].end;
tokens.push((m.index, m.captures()[0].start, end));
pos = if end > pos { end } else { pos + 1 };
}
tokens
}
#[test]
fn a_set_whose_cache_thrashes_retires_the_prefilter() {
let _limits = crate::regexec::shared_limits();
let json = std::fs::read_to_string(format!(
"{}/benches/grammars/css.json",
env!("CARGO_MANIFEST_DIR")
))
.unwrap();
let patterns = flat_grammar_patterns(&json);
let refs: Vec<&str> = patterns.iter().map(String::as_str).collect();
let mut filtered =
Scanner::with_config(&refs, &ScannerConfig::default().prefilter_warmup(0)).unwrap();
let mut plain =
Scanner::with_config(&refs, &ScannerConfig::default().prefilter(false)).unwrap();
let stylesheet = ".navbar-primary > .nav-item:first-child {\n\
background-color: oklch(0.65 0.15 250);\n\
font-family: 'Inter', -apple-system, BlinkMacSystemFont, sans-serif;\n\
transition: all 150ms cubic-bezier(0.4, 0, 0.2, 1);\n\
--custom-property: var(--color-primary, #3b82f6);\n\
@media (min-width: 768px) and (prefers-color-scheme: dark) {\n}\n";
let line = crate::scanner::OnigString::new(stylesheet);
let stats = filtered.prefilter_stats();
assert!(stats.covered > 100 && !stats.retired, "{stats:?}");
let want = tokenize(&mut plain, &line);
let got = tokenize(&mut filtered, &line);
assert_eq!(got, want);
assert!(got.len() > 200);
let stats = filtered.prefilter_stats();
assert!(
stats.retired && !stats.built && stats.own == refs.len(),
"{stats:?}"
);
assert_eq!(tokenize(&mut filtered, &line), want);
assert!(filtered.prefilter_stats().retired);
}
#[test]
fn walks_of_a_subject_searched_again_are_kept() {
let _limits = crate::regexec::shared_limits();
let patterns = [r"[a-z]+:", r"\d+", r"[a-z]+\(", "b", "a"];
let mut filtered =
Scanner::with_config(&patterns, &ScannerConfig::default().prefilter_warmup(0)).unwrap();
let mut plain =
Scanner::with_config(&patterns, &ScannerConfig::default().prefilter(false)).unwrap();
let text = "abc: 12 foo(bar) a b xyz: 7 quux(1) ba";
let line = crate::scanner::OnigString::new(text);
let steps = || SCAN_STEPS.with(|c| c.get());
let want = tokenize(&mut plain, &line);
let before = steps();
assert_eq!(tokenize(&mut filtered, &line), want);
let first = steps() - before;
assert!(first > 0);
let before = steps();
assert_eq!(tokenize(&mut filtered, &line), want);
assert_eq!(steps() - before, first);
let before = steps();
assert_eq!(tokenize(&mut filtered, &line), want);
let third = steps() - before;
assert!(third < first / 4, "{third} steps after {first}");
let before = steps();
assert_eq!(tokenize(&mut filtered, &line), want);
assert_eq!(steps() - before, third);
let other = crate::scanner::OnigString::new(text);
let before = steps();
assert_eq!(tokenize(&mut filtered, &other), want);
assert_eq!(steps() - before, first);
}
#[test]
fn no_seek_without_the_prefilter_or_with_callouts() {
let plain = onig_new_for_scanner(
b"abc",
ONIG_OPTION_CAPTURE_GROUP,
&ONIG_ENCODING_UTF8,
&OnigSyntaxOniguruma,
false,
false,
)
.unwrap();
assert_eq!(plain.1, None);
assert_eq!(seek("(*FAIL)"), None);
assert_eq!(seek("(*MAX{1})a"), None);
assert_eq!(
seek_pattern_with("abc", ONIG_OPTION_IGNORECASE),
"(?:(?-u:[Aa])(?-u:[Bb])(?-u:[Cc]))"
);
let ascii = onig_new_for_scanner(
b"abc",
ONIG_OPTION_CAPTURE_GROUP,
&crate::encodings::ascii::ONIG_ENCODING_ASCII,
&OnigSyntaxOniguruma,
false,
true,
)
.unwrap();
assert_eq!(ascii.1, None);
}
#[test]
fn sets_over_the_state_cutoff_build_no_automata() {
let _limits = crate::regexec::shared_limits();
let branches: Vec<String> = (0..16)
.map(|i| format!("[{}{}]{{64}}{i:x}", (b'a' + i) as char, (b'A' + i) as char))
.collect();
let pattern = format!("(?:{})", branches.join("|"));
let patterns: Vec<&str> = std::iter::repeat_n(pattern.as_str(), 70)
.chain(["2"])
.collect();
let text = format!("1 {}0 2", "a".repeat(64));
let config = ScannerConfig::default().prefilter_warmup(0);
let mut big = Scanner::with_config(&patterns, &config).unwrap();
let mut little = Scanner::with_config(&patterns[69..], &config).unwrap();
for scanner in [&big, &little] {
let stats = scanner.prefilter_stats();
assert!(!stats.built && stats.nfa_states == 0, "{stats:?}");
}
let found = |scanner: &mut Scanner, start: usize| {
scanner
.find_next_match(&text, start, ScannerFindOptions::NONE)
.map(|m| (m.index, m.captures()[0].start, m.captures()[0].end))
};
assert_eq!(found(&mut big, 0), Some((0, 2, 67)));
assert_eq!(found(&mut big, 3), Some((70, 68, 69)));
assert_eq!(found(&mut little, 0), Some((0, 2, 67)));
assert_eq!(found(&mut little, 3), Some((1, 68, 69)));
let stats = big.prefilter_stats();
assert!(
!stats.built && stats.covered == 0 && stats.own == 71,
"{stats:?}"
);
let stats = little.prefilter_stats();
assert!(stats.built && stats.covered == 2 && stats.nfa_states < MAX_NFA_STATES);
}
struct OverlappingCacheBytes;
impl OverlappingCacheBytes {
fn set(bytes: usize) -> Self {
OVERLAPPING_CACHE_BYTES.with(|cell| cell.set(Some(bytes)));
Self
}
}
impl Drop for OverlappingCacheBytes {
fn drop(&mut self) {
OVERLAPPING_CACHE_BYTES.with(|cell| cell.set(None));
}
}
#[test]
fn a_walk_that_clears_the_cache_keeps_the_state_it_reaches() {
let _limits = crate::regexec::shared_limits();
let patterns = ["zz", "z", r"z\w*", "q:"];
let _cache = OverlappingCacheBytes::set(2048);
let mut filtered =
Scanner::with_config(&patterns, &ScannerConfig::default().prefilter_warmup(0)).unwrap();
let mut plain =
Scanner::with_config(&patterns, &ScannerConfig::default().prefilter(false)).unwrap();
let line = crate::scanner::OnigString::new("zqzzzz(");
let want = tokenize(&mut plain, &line);
let got = tokenize(&mut filtered, &line);
let stats = filtered.prefilter_stats();
assert!(stats.built && stats.cache_clears > 0, "{stats:?}");
assert_eq!(want, [(1, 0, 1), (0, 2, 4), (0, 4, 6)]);
assert_eq!(got, want);
}
#[test]
fn single_table_matches_hybrid_at_every_byte_context() {
for pattern in [
r"",
r"a*",
r"\Aa+\z",
r"\z\A",
r"(?:ab|a)\z",
r"[a-z]+",
r"é+",
r"a\z",
r"(?:a?|b)*c",
r"[^x]+",
r"\A(?:a|bc){0,3}\z",
] {
let nfa = thompson::Compiler::new()
.configure(
thompson::Config::new()
.utf8(false)
.which_captures(thompson::WhichCaptures::None),
)
.build(pattern)
.unwrap();
let dfa = build_single_hybrid(nfa.clone()).unwrap();
let mut cache = dfa.create_cache();
let table = SingleTable::build(&nfa, PREFERRED_SINGLE_DENSE_LIMITS).unwrap();
for previous in 0..=256 {
for suffix in [
"", "a", "aa", "ab", "ba", "abc", "\n", "a\n", "a\r\n", "é", "éa",
] {
let mut text = vec![];
if previous < 256 {
text.push(previous as u8);
}
let start = text.len();
text.extend_from_slice(suffix.as_bytes());
let expected = dfa
.try_search_fwd(
&mut cache,
&Input::new(&text).span(start..text.len()).earliest(true),
)
.unwrap()
.is_some();
assert_eq!(
table.search(&text, start),
expected,
"{pattern:?} previous={previous} suffix={suffix:?}"
);
}
}
}
}
#[test]
fn single_table_guards_bitset_width_and_assertions() {
let mut saw_64 = false;
let mut saw_65 = false;
for count in 1..=80 {
let nfa = thompson::Compiler::new()
.configure(thompson::Config::new().utf8(false))
.build(&format!("a{{{count}}}"))
.unwrap();
let size = nfa.states().len();
saw_64 |= size == 64;
saw_65 |= size == 65;
assert_eq!(SingleTable::supports(&nfa), size <= 64);
if size > 64 {
assert!(SingleTable::build(&nfa, PREFERRED_SINGLE_DENSE_LIMITS).is_none());
}
if size == 64 {
let table = SingleTable::build(&nfa, PREFERRED_SINGLE_DENSE_LIMITS).unwrap();
assert!(table.search("a".repeat(count).as_bytes(), 0));
assert!(!table.search("a".repeat(count - 1).as_bytes(), 0));
}
}
assert!(saw_64 && saw_65);
for pattern in [r"(?m)^a+$", r"(?-u:\b)a", r"(?-u:\b{start-half})a"] {
let nfa = thompson::Compiler::new()
.configure(thompson::Config::new().utf8(false))
.build(pattern)
.unwrap();
assert!(!SingleTable::supports(&nfa));
assert!(SingleTable::build(&nfa, PREFERRED_SINGLE_DENSE_LIMITS).is_none());
}
}
#[test]
fn reject_only_search_keeps_the_hybrid_path_before_and_after_promotion() {
let seek = adaptive_single_seek_with(
r"\A(?:ab|cd)[0-9]{1,3}\z",
ONIG_OPTION_CAPTURE_GROUP,
Some(PREFERRED_SINGLE_DENSE_LIMITS),
2,
)
.unwrap();
for _ in 0..128 {
assert_eq!(seek.search_hybrid(b"ab1", 0), Some(true));
assert_eq!(seek.search_hybrid(b"xx", 0), Some(false));
}
assert_eq!(seek.adaptive_consultations(), (Some(0), None));
assert_eq!(seek.backend_names().0, "hybrid");
assert_eq!(seek.search(b"ab1", 0), Some(true));
assert_eq!(seek.search(b"xx", 0), Some(false));
assert_eq!(seek.backend_names().0, "dense");
for _ in 0..128 {
assert_eq!(seek.search_hybrid(b"ab1", 0), Some(true));
assert_eq!(seek.search_hybrid(b"xx", 0), Some(false));
}
assert_eq!(seek.adaptive_consultations(), (Some(2), None));
}
}