use {
crate::{look_behind::JavaWindow, translate::BackrefFold},
fancy_regex::{Assertion, Expr, LookAround},
regex::Regex,
regex_syntax::ast::{
Ast, ClassBracketed, ClassSet, ClassSetBinaryOpKind, ClassSetItem, ClassUnicode,
ClassUnicodeKind, parse::Parser,
},
std::{ops::Range, slice::Iter},
};
const BACKTRACK_LIMIT: usize = 1_000_000;
#[derive(Debug, thiserror::Error)]
pub(super) enum SearchError {
#[error("the search ran past the backtrack limit of {BACKTRACK_LIMIT} steps")]
BacktrackLimit,
}
#[derive(Debug)]
pub(super) struct Program {
blocks: Vec<Vec<Inst>>,
slot_count: usize,
groups: usize,
differs: bool,
}
#[derive(Debug)]
enum Inst {
Char(char),
Class(Class),
Any {
newline: bool,
},
Linebreak {
give_back: bool,
},
Split {
first: usize,
second: usize,
},
Jump(usize),
Save(usize),
Close {
group: usize,
open: usize,
},
Backref {
group: usize,
fold: Option<BackrefFold>,
},
Assert(Check),
Look {
block: usize,
behind: Option<Behind>,
negated: bool,
},
Atomic(usize),
Iteration {
block: usize,
group: Option<usize>,
},
RepeatStart {
count: usize,
},
Repeat {
count: usize,
lo: usize,
hi: usize,
greedy: bool,
exit: usize,
},
RepeatEnd {
count: usize,
lo: usize,
start: usize,
head: usize,
exit: usize,
group_curly: bool,
},
Match,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
enum Unit {
Char(char),
LowSurrogate(u16),
}
#[derive(Debug)]
struct Class {
ascii: u128,
surrogates: bool,
regex: Regex,
}
impl Class {
fn new(class: &str) -> Option<Self> {
let regex = Regex::new(&format!("\\A(?:{class})\\z")).ok()?;
let ascii = (0..128_u8)
.filter(|&byte| regex.is_match(char::from(byte).encode_utf8(&mut [0; 4])))
.fold(0, |table, byte| table | 1 << byte);
Some(Self {
ascii,
surrogates: class_has_surrogates(class),
regex,
})
}
fn matches(&self, c: char) -> bool {
if c.is_ascii() {
self.ascii >> u32::from(c) & 1 == 1
} else {
self.regex.is_match(c.encode_utf8(&mut [0; 4]))
}
}
fn matches_unit(&self, unit: Unit) -> bool {
match unit {
Unit::Char(c) => self.matches(c),
Unit::LowSurrogate(_) => self.surrogates,
}
}
}
fn class_has_surrogates(class: &str) -> bool {
Parser::new()
.parse(class)
.ok()
.and_then(|ast| ast_has_surrogates(&ast))
.unwrap_or(false)
}
fn ast_has_surrogates(ast: &Ast) -> Option<bool> {
match ast {
Ast::Literal(_) => Some(false),
Ast::Dot(_) => Some(true),
Ast::ClassPerl(class) => Some(class.negated),
Ast::ClassUnicode(class) => Some(unicode_has_surrogates(class)),
Ast::ClassBracketed(class) => Some(bracketed_has_surrogates(class)),
Ast::Group(group) if !group.is_capturing() => ast_has_surrogates(&group.ast),
Ast::Concat(concat) => {
let mut classes = concat
.asts
.iter()
.filter(|ast| !matches!(ast, Ast::Flags(_)));
match (classes.next(), classes.next()) {
(Some(class), None) => ast_has_surrogates(class),
_ => None,
}
}
_ => None,
}
}
fn unicode_has_surrogates(class: &ClassUnicode) -> bool {
let named = |name: &str| {
let name = normalize_property_name(name);
category_has_surrogates(&name) || script_has_surrogates(&name)
};
let surrogates = match &class.kind {
ClassUnicodeKind::OneLetter(letter) => named(&letter.to_string()),
ClassUnicodeKind::Named(name) => named(name),
ClassUnicodeKind::NamedValue { name, value, .. } => {
let value = normalize_property_name(value);
match normalize_property_name(name).as_str() {
"gc" | "generalcategory" => category_has_surrogates(&value),
"sc" | "script" | "scx" | "scriptextensions" => script_has_surrogates(&value),
_ => false,
}
}
};
surrogates != class.is_negated()
}
fn bracketed_has_surrogates(class: &ClassBracketed) -> bool {
set_has_surrogates(&class.kind) != class.negated
}
fn set_has_surrogates(set: &ClassSet) -> bool {
match set {
ClassSet::Item(item) => item_has_surrogates(item),
ClassSet::BinaryOp(op) => {
let (lhs, rhs) = (set_has_surrogates(&op.lhs), set_has_surrogates(&op.rhs));
match op.kind {
ClassSetBinaryOpKind::Intersection => lhs && rhs,
ClassSetBinaryOpKind::Difference => lhs && !rhs,
ClassSetBinaryOpKind::SymmetricDifference => lhs != rhs,
}
}
}
}
fn item_has_surrogates(item: &ClassSetItem) -> bool {
match item {
ClassSetItem::Empty(_) | ClassSetItem::Literal(_) => false,
ClassSetItem::Range(range) => {
(u32::from(range.start.c)..=u32::from(range.end.c)).contains(&0xDC00)
}
ClassSetItem::Ascii(class) => class.negated,
ClassSetItem::Perl(class) => class.negated,
ClassSetItem::Unicode(class) => unicode_has_surrogates(class),
ClassSetItem::Bracketed(class) => bracketed_has_surrogates(class),
ClassSetItem::Union(union) => union.items.iter().any(item_has_surrogates),
}
}
fn normalize_property_name(name: &str) -> String {
let (rest, is) = match name.get(..2) {
Some(prefix) if prefix.eq_ignore_ascii_case("is") => (&name[2..], true),
_ => (name, false),
};
let normalized: String = rest
.chars()
.filter(|c| c.is_ascii() && !matches!(c, ' ' | '_' | '-'))
.map(|c| c.to_ascii_lowercase())
.collect();
if is && normalized == "c" {
"isc".to_owned()
} else {
normalized
}
}
fn category_has_surrogates(name: &str) -> bool {
matches!(name, "cs" | "surrogate" | "c" | "other" | "assigned")
}
fn script_has_surrogates(name: &str) -> bool { matches!(name, "zzzz" | "unknown") }
#[derive(Clone, Copy, Debug)]
enum Behind {
Window(JavaWindow),
Lengths(Lengths),
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
struct Lengths {
min: usize,
max: Option<usize>,
}
impl Lengths {
const fn exactly(units: usize) -> Self {
Self {
min: units,
max: Some(units),
}
}
fn then(self, other: Self) -> Self {
Self {
min: self.min.saturating_add(other.min),
max: self.max.zip(other.max).map(|(a, b)| a.saturating_add(b)),
}
}
fn or(self, other: Self) -> Self {
Self {
min: self.min.min(other.min),
max: self.max.zip(other.max).map(|(a, b)| a.max(b)),
}
}
fn of(expr: &Expr) -> Self {
const UNBOUNDED: Lengths = Lengths { min: 0, max: None };
match expr {
Expr::Empty
| Expr::Assertion(_)
| Expr::LookAround(..)
| Expr::ContinueFromPreviousMatchEnd => Self::exactly(0),
Expr::Literal { val, .. } => Self::exactly(val.encode_utf16().count()),
Expr::Any { .. } | Expr::Delegate { .. } | Expr::GeneralNewline { .. } => Self {
min: 1,
max: Some(2),
},
Expr::Concat(children) => children
.iter()
.map(Self::of)
.fold(Self::exactly(0), Self::then),
Expr::Alt(children) => children
.iter()
.map(Self::of)
.reduce(Self::or)
.unwrap_or(Self::exactly(0)),
Expr::Group(child) => Self::of(child),
Expr::AtomicGroup(child) => Self::of(child),
Expr::Repeat { child, lo, hi, .. } => {
let child = Self::of(child);
Self {
min: child.min.saturating_mul(*lo),
max: child
.max
.filter(|_| *hi != usize::MAX)
.map(|max| max.saturating_mul(*hi)),
}
}
_ => UNBOUNDED,
}
}
}
#[derive(Debug)]
enum Check {
StartText,
EndText,
LastMatchEnd,
EndTextBeforeNewlines,
StartLine,
EndLine,
WordBoundary {
word: Class,
negated: bool,
},
}
impl Program {
pub(super) fn new(
translated: &str,
groups: usize,
folds: &[BackrefFold],
linebreak_atoms: &[bool],
group_repeats: &[bool],
windows: &[JavaWindow],
) -> Option<Self> {
let tree = Expr::parse_tree(translated).ok()?;
let mut compiler = Compiler {
blocks: vec![Vec::new()],
slot_count: 2 * groups,
next_group: 1,
folds: folds.iter(),
linebreak_atoms: linebreak_atoms.iter(),
group_repeats,
windows: windows.iter(),
nested: false,
first_way: false,
open_groups: Vec::new(),
differs: false,
};
let mut main = Vec::new();
compiler.compile(&tree.expr, &mut main)?;
main.push(Inst::Match);
compiler.blocks[0] = main;
if compiler.next_group != groups
|| compiler.folds.next().is_some()
|| compiler.linebreak_atoms.next().is_some()
{
return None;
}
Some(Self {
blocks: compiler.blocks,
slot_count: compiler.slot_count,
groups,
differs: compiler.differs,
})
}
pub(super) fn differs_from_fancy(&self) -> bool { self.differs }
pub(super) fn matches(&self, input: &str) -> Result<bool, SearchError> {
let mut slots = vec![None; self.slot_count];
let mut machine = Machine::new(self, input, 0);
Ok(machine.run(0, 0, Some(input.len()), &mut slots)?.is_some())
}
pub(super) fn find(
&self,
input: &str,
from: usize,
last: usize,
) -> Result<Option<Vec<Option<Range<usize>>>>, SearchError> {
let mut machine = Machine::new(self, input, last);
let starts = input[from..]
.char_indices()
.map(|(offset, _)| from + offset)
.chain([input.len()]);
let mut slots = vec![None; self.slot_count];
for start in starts {
if let Some(end) = machine.run(0, start, None, &mut slots)? {
slots[0] = Some(start);
slots[1] = Some(end);
return Ok(Some(
slots
.chunks(2)
.take(self.groups)
.map(|pair| Some(pair[0]?..pair[1]?))
.collect(),
));
}
}
Ok(None)
}
}
struct Compiler<'a> {
blocks: Vec<Vec<Inst>>,
slot_count: usize,
next_group: usize,
folds: Iter<'a, BackrefFold>,
linebreak_atoms: Iter<'a, bool>,
group_repeats: &'a [bool],
windows: Iter<'a, JavaWindow>,
nested: bool,
first_way: bool,
open_groups: Vec<usize>,
differs: bool,
}
impl Compiler<'_> {
fn slot(&mut self) -> usize {
self.slot_count += 1;
self.slot_count - 1
}
fn nested_block(&mut self, expr: &Expr) -> Option<usize> {
let outer = self.nested;
self.nested = true;
let block = self.block(expr)?;
self.nested = outer;
Some(block)
}
fn block(&mut self, expr: &Expr) -> Option<usize> {
let outer = self.first_way;
self.first_way = false;
let mut code = Vec::new();
self.compile(expr, &mut code)?;
self.first_way = outer;
code.push(Inst::Match);
self.blocks.push(code);
Some(self.blocks.len() - 1)
}
fn compile(&mut self, expr: &Expr, code: &mut Vec<Inst>) -> Option<()> {
match expr {
Expr::Empty => {}
Expr::Any {
newline,
crlf: false,
} => code.push(Inst::Any { newline: *newline }),
Expr::GeneralNewline { .. } => {
self.linebreak_atoms.next()?;
self.differs = true;
code.push(Inst::Linebreak {
give_back: !self.first_way,
});
}
Expr::ContinueFromPreviousMatchEnd => {
self.differs = true;
code.push(Inst::Assert(Check::LastMatchEnd));
}
Expr::Assertion(assertion) => code.push(Inst::Assert(check(*assertion)?)),
Expr::Literal { val, casei: false } => code.extend(val.chars().map(Inst::Char)),
Expr::Concat(children) => {
for child in children {
self.compile(child, code)?;
}
}
Expr::Alt(children) => self.alternation(children, code)?,
Expr::Group(child) => {
let group = self.next_group;
self.next_group += 1;
self.differs |= self.nested;
let open = self.slot();
code.push(Inst::Save(open));
self.open_groups.push(group);
self.compile(child, code)?;
self.open_groups.pop();
code.push(Inst::Close { group, open });
}
Expr::LookAround(child, kind) => {
let (behind, negated) = match kind {
LookAround::LookAhead => (false, false),
LookAround::LookAheadNeg => (false, true),
LookAround::LookBehind => (true, false),
LookAround::LookBehindNeg => (true, true),
};
let behind = behind.then(|| {
self.windows.next().map_or_else(
|| Behind::Lengths(Lengths::of(child)),
|&window| Behind::Window(window),
)
});
let block = self.nested_block(child)?;
code.push(Inst::Look {
block,
behind,
negated,
});
}
Expr::Repeat {
child,
lo,
hi,
greedy,
} => self.repeat(child, *lo, *hi, *greedy, code)?,
Expr::Delegate {
inner,
casei: false,
} => {
code.push(Inst::Class(Class::new(inner)?));
}
Expr::Backref { group, casei } => {
self.differs |= *casei || self.open_groups.contains(group);
let fold = if *casei {
Some(*self.folds.next()?)
} else {
None
};
code.push(Inst::Backref {
group: *group,
fold,
});
}
Expr::AtomicGroup(child) => {
let block = self.nested_block(child)?;
code.push(Inst::Atomic(block));
}
_ => return None,
}
Some(())
}
fn alternation(&mut self, children: &[Expr], code: &mut Vec<Inst>) -> Option<()> {
let Some((last, rest)) = children.split_last() else {
return Some(());
};
let mut jumps = Vec::new();
for child in rest {
let split = code.len();
code.push(Inst::Split {
first: split + 1,
second: 0,
});
self.compile(child, code)?;
jumps.push(code.len());
code.push(Inst::Jump(0));
let next = code.len();
if let Inst::Split { second, .. } = &mut code[split] {
*second = next;
}
}
self.compile(last, code)?;
let end = code.len();
for jump in jumps {
code[jump] = Inst::Jump(end);
}
Some(())
}
fn repeat(
&mut self,
child: &Expr,
lo: usize,
hi: usize,
greedy: bool,
code: &mut Vec<Inst>,
) -> Option<()> {
let (count, start) = (self.slot(), self.slot());
let group_curly = (lo, hi) != (0, 1)
&& matches!(
child,
Expr::Group(_) | Expr::Concat(_) | Expr::Repeat { .. }
)
&& deterministic(child);
let repeated = match child {
Expr::Group(body) if group_curly => self
.group_repeats
.get(self.next_group)
.copied()
.unwrap_or(false)
.then_some((self.next_group, &**body)),
_ => None,
};
self.differs |= match repeated {
Some((_, body)) => can_be_empty(child) || has_group(body),
None if group_curly => has_group(child),
None => (lo, hi) != (0, 1) && has_group(child) && can_be_empty(child),
};
code.push(Inst::RepeatStart { count });
let head = code.len();
code.push(Inst::Repeat {
count,
lo,
hi,
greedy,
exit: 0,
});
code.push(Inst::Save(start));
if group_curly {
let block = self.block(child)?;
code.push(Inst::Iteration {
block,
group: repeated.map(|(group, _)| group),
});
} else {
let outer = self.first_way;
let linebreak_atom = matches!(child, Expr::GeneralNewline { .. })
&& self.linebreak_atoms.as_slice().first() == Some(&true);
self.first_way |= takes_first_way(child, lo, hi, linebreak_atom);
self.compile(child, code)?;
self.first_way = outer;
}
let exit = code.len() + 1;
code.push(Inst::RepeatEnd {
count,
lo,
start,
head,
exit,
group_curly,
});
if let Inst::Repeat { exit: target, .. } = &mut code[head] {
*target = exit;
}
Some(())
}
}
fn takes_first_way(child: &Expr, lo: usize, hi: usize, linebreak_atom: bool) -> bool {
match child {
_ if linebreak_atom => true,
_ if (lo, hi) == (0, 1) => false,
_ => deterministic(child),
}
}
pub(super) fn deterministic(expr: &Expr) -> bool {
match expr {
Expr::Alt(_) => false,
Expr::Repeat { child, lo, hi, .. } => lo == hi && deterministic(child),
Expr::Concat(children) => children.iter().all(deterministic),
Expr::Group(child) => deterministic(child),
Expr::AtomicGroup(child) => deterministic(child),
_ => true,
}
}
fn has_group(expr: &Expr) -> bool {
match expr {
Expr::Group(_) => true,
Expr::Concat(children) | Expr::Alt(children) => children.iter().any(has_group),
Expr::Repeat { child, .. } | Expr::LookAround(child, _) | Expr::AtomicGroup(child) => {
has_group(child)
}
_ => false,
}
}
fn can_be_empty(expr: &Expr) -> bool {
match expr {
Expr::Literal { val, .. } => val.is_empty(),
Expr::Any { .. } | Expr::Delegate { .. } | Expr::GeneralNewline { .. } => false,
Expr::Concat(children) => children.iter().all(can_be_empty),
Expr::Alt(children) => children.iter().any(can_be_empty),
Expr::Group(child) => can_be_empty(child),
Expr::AtomicGroup(child) => can_be_empty(child),
Expr::Repeat { child, lo, .. } => *lo == 0 || can_be_empty(child),
_ => true,
}
}
fn check(assertion: Assertion) -> Option<Check> {
Some(match assertion {
Assertion::StartText => Check::StartText,
Assertion::EndText => Check::EndText,
Assertion::EndTextIgnoreTrailingNewlines { crlf: false } => Check::EndTextBeforeNewlines,
Assertion::StartLine { crlf: false } => Check::StartLine,
Assertion::EndLine { crlf: false } => Check::EndLine,
Assertion::WordBoundary | Assertion::NotWordBoundary => Check::WordBoundary {
word: Class::new(r"\w")?,
negated: matches!(assertion, Assertion::NotWordBoundary),
},
_ => return None,
})
}
#[derive(Clone, Copy)]
enum Frame {
Alternative { pc: usize, pos: usize },
Undo { slot: usize, value: Option<usize> },
}
struct Machine<'a> {
program: &'a Program,
input: &'a str,
last_match_end: usize,
budget: usize,
stack: Vec<Frame>,
kept: Vec<usize>,
keeps: usize,
}
impl<'a> Machine<'a> {
fn new(program: &'a Program, input: &'a str, last_match_end: usize) -> Self {
Self {
program,
input,
last_match_end,
budget: BACKTRACK_LIMIT,
stack: Vec::new(),
kept: vec![0; program.slot_count],
keeps: 0,
}
}
fn run(
&mut self,
block: usize,
at: usize,
end: Option<usize>,
slots: &mut [Option<usize>],
) -> Result<Option<usize>, SearchError> {
let program = self.program;
let code = &program.blocks[block];
let base = self.stack.len();
let (mut pc, mut pos) = (0, at);
loop {
let next = match &code[pc] {
Inst::Char(expected) => self
.next_unit(pos)
.filter(|&(actual, _)| actual == Unit::Char(*expected))
.map(|(_, after)| (pc + 1, after)),
Inst::Class(class) => self
.next_unit(pos)
.filter(|&(actual, _)| class.matches_unit(actual))
.map(|(_, after)| (pc + 1, after)),
Inst::Any { newline } => self
.next_unit(pos)
.filter(|&(actual, _)| *newline || actual != Unit::Char('\n'))
.map(|(_, after)| (pc + 1, after)),
&Inst::Linebreak { give_back } => {
if self
.input
.get(pos..)
.is_some_and(|after| after.starts_with("\r\n"))
{
if give_back {
self.stack.push(Frame::Alternative {
pc: pc + 1,
pos: pos + 1,
});
}
Some((pc + 1, pos + 2))
} else {
self.next_unit(pos)
.filter(|(actual, _)| {
matches!(
actual,
Unit::Char(
'\n' | '\x0B'
| '\x0C'
| '\r'
| '\u{85}'
| '\u{2028}'
| '\u{2029}'
)
)
})
.map(|(_, after)| (pc + 1, after))
}
}
&Inst::Split { first, second } => {
self.stack.push(Frame::Alternative { pc: second, pos });
Some((first, pos))
}
&Inst::Jump(target) => Some((target, pos)),
&Inst::Save(slot) => {
set(&mut self.stack, slots, slot, Some(pos));
Some((pc + 1, pos))
}
&Inst::Close { group, open } => {
set(&mut self.stack, slots, 2 * group, slots[open]);
set(&mut self.stack, slots, 2 * group + 1, Some(pos));
Some((pc + 1, pos))
}
&Inst::Backref { group, fold } => slots[2 * group]
.zip(slots[2 * group + 1])
.and_then(|(start, stop)| self.backref(pos, start..stop, fold))
.map(|after| (pc + 1, after)),
Inst::Assert(check) => self.holds(check, pos).then_some((pc + 1, pos)),
&Inst::Look {
block,
behind,
negated,
} => {
let nested = self.stack.len();
let found = match behind {
Some(behind) => self.behind(block, pos, behind, slots)?,
None => self.run(block, pos, None, slots)?.is_some(),
};
self.keep(nested, slots, None);
(found != negated).then_some((pc + 1, pos))
}
&Inst::Atomic(block) => {
let nested = self.stack.len();
let after = self.run(block, pos, None, slots)?;
self.keep(nested, slots, None);
after.map(|after| (pc + 1, after))
}
&Inst::Iteration { block, group } => {
let nested = self.stack.len();
let after = self.run(block, pos, None, slots)?;
self.keep(nested, slots, group);
after.map(|after| (pc + 1, after))
}
&Inst::RepeatStart { count } => {
set(&mut self.stack, slots, count, Some(0));
Some((pc + 1, pos))
}
&Inst::Repeat {
count,
lo,
hi,
greedy,
exit,
} => {
let done = slots[count].unwrap_or(0);
if done < lo {
Some((pc + 1, pos))
} else if done >= hi {
Some((exit, pos))
} else if greedy {
self.stack.push(Frame::Alternative { pc: exit, pos });
Some((pc + 1, pos))
} else {
self.stack.push(Frame::Alternative { pc: pc + 1, pos });
Some((exit, pos))
}
}
&Inst::RepeatEnd {
count,
lo,
start,
head,
exit,
group_curly,
} => {
self.spend()?;
let done = slots[count].unwrap_or(0) + 1;
set(&mut self.stack, slots, count, Some(done));
let empty = slots[start] == Some(pos);
match (group_curly, empty) {
(true, true) if done > lo => None,
(false, true) if done >= lo => Some((exit, pos)),
_ => Some((head, pos)),
}
}
Inst::Match if end.is_none_or(|end| end == pos) => return Ok(Some(pos)),
Inst::Match => None,
};
(pc, pos) = match next {
Some(next) => next,
None => match self.backtrack(base, slots)? {
Some(next) => next,
None => return Ok(None),
},
};
}
}
fn backtrack(
&mut self,
base: usize,
slots: &mut [Option<usize>],
) -> Result<Option<(usize, usize)>, SearchError> {
while self.stack.len() > base {
match self.stack.pop() {
Some(Frame::Undo { slot, value }) => slots[slot] = value,
Some(Frame::Alternative { pc, pos }) => {
self.spend()?;
return Ok(Some((pc, pos)));
}
None => break,
}
}
Ok(None)
}
fn spend(&mut self) -> Result<(), SearchError> {
self.budget = self
.budget
.checked_sub(1)
.ok_or(SearchError::BacktrackLimit)?;
Ok(())
}
fn behind(
&mut self,
block: usize,
pos: usize,
behind: Behind,
slots: &mut [Option<usize>],
) -> Result<bool, SearchError> {
let window = match behind {
Behind::Window(window) => window,
Behind::Lengths(lengths) => return self.behind_within(block, pos, lengths, slots),
};
let (nearest, farthest) = self.window_units(pos, window);
let Some(mut start) = self.units_back(pos, nearest) else {
return Ok(false);
};
let mut back = nearest;
while back <= farthest {
self.spend()?;
if self.run(block, start, Some(pos), slots)?.is_some() {
return Ok(true);
}
let step = if window.code_points && back < farthest && start > 0 {
i64::from(self.count_chars(start, -1))
} else {
1
};
let Some(next) = self.units_back(start, step) else {
break;
};
(start, back) = (next, back + step);
}
Ok(false)
}
fn window_units(&self, pos: usize, window: JavaWindow) -> (i64, i64) {
let JavaWindow {
min,
max,
code_points,
} = window;
if code_points {
return (
i64::from(self.count_chars(pos, min.wrapping_neg())),
i64::from(self.count_chars(pos, max.wrapping_neg())),
);
}
if min >= 0 && max >= 0 {
return (i64::from(min), i64::from(max));
}
let i = self.utf16_index(pos);
let from = i.wrapping_sub(max).max(0);
let first = i.wrapping_sub(min).min(i);
(
i64::from(i) - i64::from(first),
i64::from(i) - i64::from(from),
)
}
fn behind_within(
&mut self,
block: usize,
pos: usize,
lengths: Lengths,
slots: &mut [Option<usize>],
) -> Result<bool, SearchError> {
let (mut start, mut units) = (pos, 0);
while lengths.max.is_none_or(|max| units <= max) {
if units >= lengths.min && self.input.is_char_boundary(start) {
self.spend()?;
if self.run(block, start, Some(pos), slots)?.is_some() {
return Ok(true);
}
}
let Some((_, before)) = unit_before(self.input, start) else {
break;
};
(start, units) = (before, units + 1);
}
Ok(false)
}
fn units_back(&self, pos: usize, units: i64) -> Option<usize> {
let mut at = pos;
for _ in 0..units {
at = unit_before(self.input, at)?.1;
}
Some(at)
}
fn count_chars(&self, pos: usize, length: i32) -> i32 {
let input = self.input;
if length == 1 && unit_at(input, pos).is_some_and(|(unit, _)| !is_high_surrogate(unit)) {
return 1;
}
let (mut at, mut units) = (pos, 0_i32);
if length >= 0 {
for _ in 0..length {
let Some((unit, after)) = unit_at(input, at) else {
break;
};
(at, units) = (after, units + 1);
if is_high_surrogate(unit)
&& let Some((next, after)) = unit_at(input, at)
&& is_low_surrogate(next)
{
(at, units) = (after, units + 1);
}
}
} else {
for _ in 0..length.wrapping_neg() {
let Some((unit, before)) = unit_before(input, at) else {
break;
};
(at, units) = (before, units + 1);
if is_low_surrogate(unit)
&& let Some((previous, before)) = unit_before(input, at)
&& is_high_surrogate(previous)
{
(at, units) = (before, units + 1);
}
}
}
units
}
fn utf16_index(&self, pos: usize) -> i32 {
let (boundary, inside) = if self.input.is_char_boundary(pos) {
(pos, 0)
} else {
(pos - 2, 1)
};
let units: usize = self.input[..boundary].chars().map(char::len_utf16).sum();
i32::try_from(units + inside).unwrap_or(i32::MAX)
}
fn keep(&mut self, base: usize, slots: &[Option<usize>], group: Option<usize>) {
self.keeps += 1;
let private = 2 * self.program.groups;
let mut kept = base;
for index in base..self.stack.len() {
if let Frame::Undo { slot, value } = self.stack[index]
&& (slot >= private || group.is_some_and(|group| slot / 2 == group))
&& self.kept[slot] != self.keeps
{
self.kept[slot] = self.keeps;
if slots[slot] != value {
self.stack[kept] = Frame::Undo { slot, value };
kept += 1;
}
}
}
self.stack.truncate(kept);
}
fn backref(&self, pos: usize, group: Range<usize>, fold: Option<BackrefFold>) -> Option<usize> {
let input = self.input;
if group.is_empty() {
return Some(pos);
}
let whole = [pos, group.start, group.end]
.into_iter()
.all(|at| input.is_char_boundary(at));
match fold {
None if whole => input[pos..]
.starts_with(&input[group.clone()])
.then_some(pos + group.len()),
None => {
let (mut at, mut from) = (pos, group.start);
while from < group.end {
let (expected, next_from) = unit_at(input, from)?;
let (actual, next_at) = unit_at(input, at)?;
if actual != expected {
return None;
}
(at, from) = (next_at, next_from);
}
Some(at)
}
Some(fold) if whole => folded_backref(input, pos, group, fold),
Some(_) => None,
}
}
fn next_unit(&self, pos: usize) -> Option<(Unit, usize)> {
if self.input.is_char_boundary(pos) {
let c = self.input[pos..].chars().next()?;
Some((Unit::Char(c), pos + c.len_utf8()))
} else {
let (unit, after) = unit_at(self.input, pos)?;
Some((Unit::LowSurrogate(unit), after))
}
}
fn holds(&self, check: &Check, pos: usize) -> bool {
if !self.input.is_char_boundary(pos) {
return match check {
Check::LastMatchEnd => pos == self.last_match_end,
Check::WordBoundary { negated, .. } => *negated,
_ => false,
};
}
let (before, after) = self.input.split_at(pos);
match check {
Check::StartText => before.is_empty(),
Check::EndText => after.is_empty(),
Check::LastMatchEnd => pos == self.last_match_end,
Check::EndTextBeforeNewlines => after.bytes().all(|byte| byte == b'\n'),
Check::StartLine => before.is_empty() || before.ends_with('\n'),
Check::EndLine => after.is_empty() || after.starts_with('\n'),
Check::WordBoundary { word, negated } => {
let is_word = |c: Option<char>| c.is_some_and(|c| word.matches(c));
(is_word(before.chars().next_back()) != is_word(after.chars().next())) != *negated
}
}
}
}
fn unit_at(input: &str, pos: usize) -> Option<(u16, usize)> {
if input.is_char_boundary(pos) {
let c = input[pos..].chars().next()?;
let [first, _] = utf16_units(c);
let after = if c.len_utf16() == 2 {
pos + 2
} else {
pos + c.len_utf8()
};
Some((first, after))
} else {
let c = input[pos - 2..].chars().next()?;
Some((utf16_units(c)[1], pos + 2))
}
}
fn unit_before(input: &str, pos: usize) -> Option<(u16, usize)> {
if input.is_char_boundary(pos) {
let c = input[..pos].chars().next_back()?;
let [first, second] = utf16_units(c);
Some(if c.len_utf16() == 2 {
(second, pos - 2)
} else {
(first, pos - c.len_utf8())
})
} else {
let c = input[pos - 2..].chars().next()?;
Some((utf16_units(c)[0], pos - 2))
}
}
fn utf16_units(c: char) -> [u16; 2] {
let mut units = [0; 2];
c.encode_utf16(&mut units);
units
}
const fn is_high_surrogate(unit: u16) -> bool { matches!(unit, 0xD800..=0xDBFF) }
const fn is_low_surrogate(unit: u16) -> bool { matches!(unit, 0xDC00..=0xDFFF) }
fn set(stack: &mut Vec<Frame>, slots: &mut [Option<usize>], slot: usize, value: Option<usize>) {
stack.push(Frame::Undo {
slot,
value: slots[slot],
});
slots[slot] = value;
}
fn folded_backref(input: &str, at: usize, group: Range<usize>, fold: BackrefFold) -> Option<usize> {
let mut rest = input[at..].chars();
let mut after = at;
for c2 in input[group].chars() {
let c1 = rest.next()?;
if !fold.chars_match(c2, c1) {
return None;
}
after += c1.len_utf8();
}
Some(after)
}
#[cfg(test)]
mod tests {
use {super::*, crate::next_search_start, assert2::check};
struct Case<'a> {
pattern: &'a str,
input: &'a str,
group_repeats: &'a [bool],
expected: bool,
}
const fn row<'a>(
pattern: &'a str,
input: &'a str,
group_repeats: &'a [bool],
expected: bool,
) -> Case<'a> {
Case {
pattern,
input,
group_repeats,
expected,
}
}
fn program(pattern: &str, group_repeats: &[bool], linebreak_atoms: &[bool]) -> Program {
let compiled = Program::new(
pattern,
group_repeats.len(),
&[],
linebreak_atoms,
group_repeats,
&[],
);
compiled.unwrap_or_else(|| panic!("{pattern:?} does not compile"))
}
fn check_matches<'a>(cases: impl IntoIterator<Item = Case<'a>>, linebreak_atoms: &[bool]) {
for Case {
pattern,
input,
group_repeats,
expected,
} in cases
{
let program = program(pattern, group_repeats, linebreak_atoms);
check!(
program.matches(input).ok() == Some(expected),
"{pattern:?} against {input:?}"
);
}
}
#[test]
fn a_group_curly_takes_no_empty_iteration_past_its_least_count() {
const REPEATED: &[bool] = &[false, true];
const OUTER_REPEATED: &[bool] = &[false, true, false];
const NONE_REPEATED: &[bool] = &[false, false];
let cases = [
row(r"()*\1", "", REPEATED, false),
row(r"()+\1", "", REPEATED, true),
row(r"(){2}\1", "", REPEATED, true),
row(r"(){0,3}\1", "", REPEATED, false),
row(r"()*?\1", "", REPEATED, false),
row(r"(?:())*\1", "", NONE_REPEATED, true),
row(r"(())*\2", "", OUTER_REPEATED, true),
row(r"(())*\1", "", OUTER_REPEATED, false),
row(r"(())*?\2", "", OUTER_REPEATED, false),
row(r"(())*?\1", "", OUTER_REPEATED, false),
row(r"(())*?x\2", "x", OUTER_REPEATED, false),
row(r"(()){0,1}\1", "", OUTER_REPEATED, true),
row(r"(()){0,2}\1", "", OUTER_REPEATED, false),
row(r"(()){1,3}\1", "", OUTER_REPEATED, true),
row(r"((?:)){2,}\1", "", REPEATED, true),
row(r"x()*\1", "x", REPEATED, false),
row(r"(\b)*\1", "", REPEATED, false),
row(r"(a|)*\1", "aa", REPEATED, true),
row(r"(b*)*\1", "", REPEATED, true),
row(r"(x?)*\1", "", REPEATED, true),
];
check_matches(cases, &[]);
}
#[test]
fn the_groups_inside_a_group_curly_keep_their_text_when_the_match_backtracks() {
const REPEATED: &[bool] = &[false, true];
const OUTER_REPEATED: &[bool] = &[false, true, false];
const NONE_REPEATED: &[bool] = &[false, false];
const NONE_OF_TWO_REPEATED: &[bool] = &[false, false, false];
let cases = [
row(r"(?:(?:(a))*y|a)\1", "aa", NONE_REPEATED, true),
row(r"(?:(())*?y|x)\2", "x", OUTER_REPEATED, true),
row(r"(?:(())*?y|x)\1", "x", OUTER_REPEATED, false),
row(r"(())*+\1", "", NONE_OF_TWO_REPEATED, true),
row(r"(?:((a))*y|a)\2", "aa", OUTER_REPEATED, true),
row(r"(?:(a)*y|a)\1", "aa", REPEATED, false),
row(r"(?:((a))*?y|a)\2", "aa", OUTER_REPEATED, true),
row(r"(?:((a))+y|a)\2", "aa", OUTER_REPEATED, true),
row(r"(?:(?>((a))*)y|a)\2", "aa", OUTER_REPEATED, true),
row(r"(?:((a))*+y|a)\2", "aa", NONE_OF_TWO_REPEATED, true),
row(r"(?:((a)|b)*y|a)\2", "aa", OUTER_REPEATED, false),
row(r"(?:(?:(a){2})*y|aa)\1", "aaa", REPEATED, true),
];
check_matches(cases, &[]);
}
#[yare::parameterized(
group = { r"(\R\n)*", "\r\n", &[false, true], false, true },
non_capturing_group = { r"(?:\R\n)*", "\r\n", &[false], false, true },
after_a_char = { r"(?:a\R\n)*", "a\r\n", &[false], false, true },
in_a_counted_group = { r"(?:(?:\R\n){2})*", "\r\n\r\n", &[false], false, true },
alone = { r"\R*\n", "\r\n", &[false], true, false },
alone_in_a_group = { r"(?:\R)*\n", "\r\n", &[false], false, false },
)]
fn a_linebreak_in_a_group_curly_gives_back_its_newline(
pattern: &str,
input: &str,
group_repeats: &[bool],
linebreak_atom: bool,
expected: bool,
) {
check_matches([row(pattern, input, group_repeats, expected)], &[
linebreak_atom,
]);
}
#[yare::parameterized(
one_char_group = { r"(a)*", &[false, true], false },
two_char_group = { r"(ab)+", &[false, true], false },
restored_group = { r"(?:(a)*y|a)\1", &[false, true], false },
empty_group = { r"()*", &[false, true], true },
nested_group = { r"(?:((a))*y|a)\2", &[false, true, false], true },
)]
fn a_group_curly_differs_from_fancy_only_where_it_shows(
pattern: &str,
group_repeats: &[bool],
expected: bool,
) {
check!(program(pattern, group_repeats, &[]).differs_from_fancy() == expected);
}
#[test]
fn a_look_behind_on_a_long_input_answers() {
let xs = "x".repeat(20_000);
let after_y = format!("y{xs}");
let cases = [
(r"y(?:(?<!z)x|\R)*", after_y.as_str(), &[false][..], true),
(r"x*(?<!x)", &xs, &[], false),
(r"x*(?<=y)", &xs, &[], false),
(r"x*(?<!y|x{1,3})", &xs, &[], false),
(r"x*(?<!x{1,3}y)", &xs, &[], true),
(r"x*(?<=x{2,4})", &xs, &[], true),
];
for (pattern, input, linebreak_atoms, expected) in cases {
check_matches([row(pattern, input, &[false], expected)], linebreak_atoms);
}
}
#[yare::parameterized(
any = { r".(?<=.)x", "😀x", true },
class = { r".(?<=[^a])x", "😀x", true },
negated = { r".(?<!.)x", "😀x", false },
variable_length = { r"a.(?<=a.{0,2})x", "a😀x", true },
)]
fn a_look_behind_reaches_back_over_a_supplementary_char(
pattern: &str,
input: &str,
expected: bool,
) {
check_matches([row(pattern, input, &[false], expected)], &[]);
}
#[yare::parameterized(
empty = { "", 0, Some(0) },
literal = { "ab😀", 4, Some(4) },
any = { ".", 1, Some(2) },
class = { "[a-z]", 1, Some(2) },
linebreak = { r"\R", 1, Some(2) },
concat = { "a.b", 3, Some(4) },
alternation = { "ab|c|", 0, Some(2) },
group = { "(ab)", 2, Some(2) },
atomic_group = { "(?>ab)", 2, Some(2) },
counted = { "(?:ab){2,3}", 4, Some(6) },
unbounded = { "a+", 1, None },
assertions = { r"^\b(?=abc)(?<!x)a$", 1, Some(1) },
backref = { r"(a)\1", 1, None },
)]
fn the_lengths_of_a_look_behind_bound_its_body(pattern: &str, min: usize, max: Option<usize>) {
let tree = Expr::parse_tree(pattern).unwrap_or_else(|_| panic!("{pattern:?} parses"));
check!(Lengths::of(&tree.expr) == Lengths { min, max });
}
#[yare::parameterized(
range = { "[a-z0-9._-]" },
negated = { "[^a-z]" },
case_insensitive = { "(?i:[a-z])" },
case_insensitive_kelvin = { "(?i:k)" },
word = { r"\w" },
unicode_class = { r"\p{Lu}" },
)]
fn a_class_matches_each_char_as_its_regex_does(class: &str) {
let compiled = Class::new(class).unwrap_or_else(|| panic!("{class:?} compiles"));
let regex = Regex::new(&format!(r"\A(?:{class})\z"))
.unwrap_or_else(|_| panic!("{class:?} compiles"));
for c in ('\0'..='\u{3FF}').chain(['\u{212A}', '\u{1F600}']) {
check!(
compiled.matches(c) == regex.is_match(c.encode_utf8(&mut [0; 4])),
"{class:?} against {c:?}"
);
}
}
const DOT: &str = r"[^\n\r\x{85}\x{2028}\x{2029}]";
const fn units(min: i32, max: i32) -> JavaWindow {
JavaWindow {
min,
max,
code_points: false,
}
}
const fn code_points(min: i32, max: i32) -> JavaWindow {
JavaWindow {
min,
max,
code_points: true,
}
}
fn windowed(pattern: &str, groups: usize, windows: &[JavaWindow]) -> Program {
let pattern = pattern.replace('.', DOT);
let compiled = Program::new(&pattern, groups, &[], &[], &vec![false; groups], windows);
compiled.unwrap_or_else(|| panic!("{pattern:?} does not compile"))
}
fn check_marks(pattern: &str, windows: &[JavaWindow], input: &str, expected: &str) {
check!(
mark_matches(&windowed(pattern, 1, windows), input) == expected,
"{pattern:?} against {input:?}"
);
}
fn mark_matches(program: &Program, input: &str) -> String {
let (mut out, mut copied, mut from) = (String::new(), 0, 0);
while from <= input.len() {
let groups = match program.find(input, from, copied) {
Ok(Some(groups)) => groups,
Ok(None) => break,
Err(error) => panic!("{error}"),
};
let Some(whole) = groups[0].clone() else {
break;
};
out.push_str(&input[copied..whole.start]);
out.push('<');
out.push_str(&input[whole.clone()]);
out.push('>');
copied = whole.end;
from = next_search_start(input, &whole);
}
out.push_str(&input[copied..]);
out
}
#[yare::parameterized(
dot_after_a_char = { r"a.(?<=a.)x", &[units(2, 2)], "a😀x", "a😀x" },
two_dots_over_two_pairs = { r".(?<=..)x", &[units(2, 2)], "😀😀x", "😀😀x" },
two_dots_over_a_pair_and_a_char = { r".(?<=..)x", &[units(2, 2)], "😀bx", "😀<bx>" },
two_dots_alone = { r"(?<=..)x", &[units(2, 2)], "a😀x", "a😀x" },
two_dots_alone_over_bmp = { r"(?<=..)x", &[units(2, 2)], "😀bx", "😀b<x>" },
counted_dots = { r"(?<=.{2})x", &[units(2, 2)], "😀x𝄞x", "😀x𝄞x" },
dots_with_a_range_of_counts = { r"(?<=.{1,2})x", &[units(1, 2)], "😀x𝄞x", "😀<x>𝄞<x>" },
symbol_class = { r"(?<=\p{So})x", &[units(1, 1)], "😀x𝄞x", "😀x𝄞x" },
dot_then_symbol_class = { r".(?<=\p{So})x", &[units(1, 1)], "😀x", "😀x" },
supplementary_escape = { r".(?<=\x{1F600})x", &[units(1, 1)], "😀😀x", "😀😀x" },
supplementary_class = { r"(?<=[\x{1F600}])x", &[units(1, 1)], "a😀x", "a😀x" },
negated_symbol_class = { r"(?<!\p{So})x", &[units(1, 1)], "😀x𝄞x", "😀<x>𝄞<x>" },
dot_then_negated_symbol_class = { r".(?<!\p{So})x", &[units(1, 1)], "😀😀x", "😀<😀x>" },
char_then_dot = { r"(?<=a.)x", &[units(2, 2)], "a😀x", "a😀x" },
negated_char_then_dot = { r"(?<!a.)x", &[units(2, 2)], "a😀x", "a😀<x>" },
start_then_dot = { r"(?<=^.)x", &[units(1, 1)], "😀x", "😀x" },
start_then_two_dots = { r"(?<=^..)x", &[units(2, 2)], "😀bx", "😀bx" },
dot_then_escape = { r"(?<=.\x{1F600})x", &[units(2, 2)], "😀😀x", "😀😀x" },
two_negated_classes = { r"(?<=[^a][^a])x", &[units(2, 2)], "😀x𝄞x", "😀x𝄞x" },
two_negated_classes_over_bmp = { r"(?<=[^a][^a])x", &[units(2, 2)], "😀bx", "😀b<x>" },
dot_inside_a_pair = { r".(?<=.)x", &[units(1, 1)], "😀x𝄞x", "<😀x><𝄞x>" },
dot_alone_inside_a_pair = { r"(?<=.)x", &[units(1, 1)], "a😀x", "a😀<x>" },
negated_class_inside_a_pair = { r".(?<=[^a])x", &[units(1, 1)], "a😀x", "a<😀x>" },
non_word_inside_a_pair = { r"(?<=[^0-9A-Za-z_])x", &[units(1, 1)], "😀x", "😀<x>" },
non_letter_inside_a_pair = { r"(?<=\P{L})x", &[units(1, 1)], "😀x", "😀<x>" },
letter_inside_a_pair = { r"(?<=\p{L})x", &[units(1, 1)], "😀x", "😀x" },
surrogate_range_inside_a_pair = { r"(?<=[\x{D000}-\x{E000}])x", &[units(1, 1)], "😀x", "😀<x>" },
surrogate_category_inside_a_pair = { r"(?<=\p{Cs})x", &[units(1, 1)], "😀x", "😀<x>" },
not_surrogate_inside_a_pair = { r"(?<=\P{Cs})x", &[units(1, 1)], "😀x", "😀x" },
intersection_inside_a_pair = { r"(?<=[[[a]&&[[^b]]]])x", &[units(1, 1)], "😀x", "😀x" },
negated_intersection_inside_a_pair = { r"(?<=[^[[a]&&[[^b]]]])x", &[units(1, 1)], "😀x", "😀<x>" },
negated_dot = { r"(?<!.)x", &[units(1, 1)], "😀x", "😀x" },
dot_then_negated_dot = { r".(?<!.)x", &[units(1, 1)], "😀x", "😀x" },
greedy_class = { r"(?<=[^x]+)x", &[units(1, i32::MAX)], "😀x𝄞x", "😀<x>𝄞<x>" },
empty_after_a_dot = { r"(?<=.)", &[units(1, 1)], "😀x𝄞x", "😀<>x<>𝄞<>x<>" },
two_look_behinds = { r"(?<=.)(?<=.)x", &[units(1, 1), units(1, 1)], "a😀x", "a😀<x>" },
nested_look_behind_before = { r"(?<=(?<=.).)x", &[units(1, 1), units(1, 1)], "😀x", "😀x" },
nested_look_behind_after = { r"(?<=.(?<=.))x", &[units(1, 1), units(1, 1)], "😀x", "😀<x>" },
look_ahead_inside_a_pair = { r"(?<=(?=.).)x", &[units(1, 1)], "😀x", "😀<x>" },
look_ahead_at_x_inside_a_pair = { r"(?<=(?=x).)x", &[units(1, 1)], "😀x", "😀x" },
start_of_text_inside_a_pair = { r"(?<=\A.)x", &[units(1, 1)], "😀x", "😀x" },
end_of_text_inside_a_pair = { r"(?<=\z.)x", &[units(1, 1)], "😀x", "😀x" },
word_boundary_inside_a_pair = { r"(?<=\b.)x", &[units(1, 1)], "😀x", "😀x" },
word_boundary_after_a_pair = { r"(?<=\b.)x", &[units(1, 1)], "😀bx", "😀b<x>" },
not_word_boundary_inside_a_pair = { r"(?<=\B.)x", &[units(1, 1)], "a😀x", "a😀<x>" },
not_word_boundary_after_a_pair = { r"(?<=\B.)x", &[units(1, 1)], "😀bx", "😀bx" },
)]
fn a_look_behind_tries_the_starts_of_javas_window(
pattern: &str,
windows: &[JavaWindow],
input: &str,
expected: &str,
) {
check_marks(pattern, windows, input, expected);
}
#[yare::parameterized(
word_boundary = {
r"(?<=(?:(?<=[0-9A-Za-z_])(?![0-9A-Za-z_])|(?<![0-9A-Za-z_])(?=[0-9A-Za-z_])).)x",
"b😀x",
"b😀x",
},
word_boundary_after_a_pair = {
r"(?<=(?:(?<=[0-9A-Za-z_])(?![0-9A-Za-z_])|(?<![0-9A-Za-z_])(?=[0-9A-Za-z_])).)x",
"😀bx",
"😀b<x>",
},
not_word_boundary = {
r"(?<=(?:(?<=[0-9A-Za-z_])(?=[0-9A-Za-z_])|(?<![0-9A-Za-z_])(?![0-9A-Za-z_])).)x",
"😀x𝄞x",
"😀<x>𝄞<x>",
},
end_of_line = {
r"(?<=(?:(?=(?:\r\n|(?<!\r)\n|[\r\x{85}\x{2028}\x{2029}])?\z)).)x",
"😀x",
"😀x",
},
start_of_line = {
r"(?<=(?:(?!\z)(?:\A|(?<=[\n\x{85}\x{2028}\x{2029}])|(?<=\r)(?!\n))).)x",
"a😀x",
"a😀x",
},
)]
fn a_rewritten_assertion_inside_a_pair_reads_as_javas(
pattern: &str,
input: &str,
expected: &str,
) {
let look_behinds = pattern.matches("(?<").count();
let windows = vec![units(1, 1); look_behinds];
check!(
mark_matches(&windowed(pattern, 1, &windows), input) == expected,
"{pattern:?} against {input:?}"
);
}
#[yare::parameterized(
pair = { r"(?<=😀)x", &[code_points(1, 1)], "😀x𝄞x", "😀<x>𝄞x" },
pair_before_a_char = { r"(?<=😀)x", &[code_points(1, 1)], "😀bx", "😀bx" },
dot_then_pair = { r".(?<=😀)x", &[code_points(1, 1)], "😀😀x", "😀<😀x>" },
char_then_pair = { r"(?<=a😀)x", &[code_points(2, 2)], "a😀x", "a😀<x>" },
pair_then_dot = { r"(?<=😀.)x", &[code_points(2, 2)], "😀😀x", "😀😀<x>" },
pair_then_dot_over_bmp = { r"(?<=😀.)x", &[code_points(2, 2)], "😀a😀x", "😀a😀x" },
dot_then_pair_alone = { r"(?<=.😀)x", &[code_points(2, 2)], "a😀x", "a😀<x>" },
dot_then_pair_at_the_start = { r"(?<=.😀)x", &[code_points(2, 2)], "😀x", "😀x" },
negated_class_then_pair = { r"(?<=[^a]😀)x", &[code_points(2, 2)], "😀😀x", "😀😀<x>" },
negated_class_then_pair_over_a = { r"(?<=[^a]😀)x", &[code_points(2, 2)], "😀a😀x", "😀a😀x" },
negated_pair = { r"(?<!😀)x", &[code_points(1, 1)], "😀x𝄞x", "😀x𝄞<x>" },
negated_dot_then_pair = { r"(?<!.😀)x", &[code_points(2, 2)], "😀x", "😀<x>" },
negated_dot_then_pair_after_a = { r"(?<!.😀)x", &[code_points(2, 2)], "a😀x", "a😀x" },
two_dots_then_pair = { r"(?<=..😀)x", &[code_points(3, 3)], "😀a😀x", "😀a😀<x>" },
two_dots_then_pair_over_pairs = { r"(?<=..😀)x", &[code_points(3, 3)], "😀😀x", "😀😀x" },
counted_dots_then_pair = { r"(?<=.{1,2}😀)x", &[code_points(2, 3)], "😀a😀x", "😀a😀<x>" },
counted_dots_then_pair_at_the_start = { r"(?<=.{1,2}😀)x", &[code_points(2, 3)], "😀x", "😀x" },
alternation = { r"(?<=😀|a)x", &[code_points(1, 1)], "😀x𝄞x", "😀<x>𝄞x" },
two_look_behinds = { r"(?<=😀)(?<=.)x", &[code_points(1, 1), code_points(1, 1)], "a😀x", "a😀<x>" },
)]
fn a_look_behind_with_a_supplementary_char_counts_code_points(
pattern: &str,
windows: &[JavaWindow],
input: &str,
expected: &str,
) {
check_marks(pattern, windows, input, expected);
}
#[yare::parameterized(
dot_starts_inside_a_pair = { r"(?<=(.))x", "😀x", &[Some(4..5), Some(2..4)] },
dot_after_a_pair_and_a_char = { r"(?<=(.))x", "😀x𝄞x", &[Some(4..5), Some(2..4)] },
empty_group_inside_a_pair = { r"(?<=().)x", "😀x", &[Some(4..5), Some(2..2)] },
reference_to_an_empty_group = { r"(?<=().)x\1", "😀x", &[Some(4..5), Some(2..2)] },
reference_to_a_low_surrogate = { r"(?<=(.))x\1", "😀x", &[] },
group_before_the_match = { r"(?<=(.)).", "😀a😀x", &[Some(4..5), Some(2..4)] },
group_of_a_char = { r"(?<=(.)).", "a😀x", &[Some(1..5), Some(0..1)] },
)]
fn a_look_behind_sets_a_group_inside_a_pair(
pattern: &str,
input: &str,
expected: &[Option<Range<usize>>],
) {
let program = windowed(pattern, 2, &[units(1, 1)]);
let found = program.find(input, 0, 0).ok().flatten().unwrap_or_default();
check!(found == expected, "{pattern:?} against {input:?}");
}
#[yare::parameterized(
back_over_a_pair = { "😀x", 4, -1, 2 },
back_over_a_pair_and_a_char = { "😀x", 5, -2, 3 },
back_from_inside_a_pair = { "😀x", 2, -1, 1 },
back_past_the_start = { "a😀", 5, -5, 3 },
back_from_the_start = { "😀", 0, -1, 0 },
back_none = { "😀", 4, 0, 0 },
forward_over_a_pair = { "😀x", 0, 1, 2 },
forward_from_inside_a_pair = { "😀x", 2, 1, 1 },
forward_past_the_end = { "😀x", 0, 5, 3 },
most_negative = { "😀x", 5, i32::MIN, 0 },
)]
fn count_chars_counts_as_javas(input: &str, pos: usize, length: i32, expected: i32) {
let program = windowed("", 1, &[]);
check!(Machine::new(&program, input, 0).count_chars(pos, length) == expected);
}
#[yare::parameterized(
rmin_below_zero = { units(-1, 1), "ab", true },
rmax_below_zero = { units(0, -1), "ab", false },
rmax_wraps = { units(0, i32::MIN + 1), "ab", true },
)]
fn a_wrapped_window_counts_as_javas_int(window: JavaWindow, input: &str, expected: bool) {
let program = windowed(r"ab(?<=b|)", 1, &[window]);
check!(program.matches(input).ok() == Some(expected));
}
#[yare::parameterized(
dot = { ".", true },
negated_char = { "[^a]", true },
negated_char_under_a_case_flag = { "[^aA]", true },
non_word = { "[^0-9A-Za-z_]", true },
non_space = { r"[^ \t\n\x0B\x0C\r]", true },
non_digit = { "[^0-9]", true },
word = { "[0-9A-Za-z_]", false },
digit = { "[0-9]", false },
space = { r"[ \t\n\x0B\x0C\r]", false },
unicode_word = { r"\w", false },
unicode_non_word = { r"\W", true },
non_horizontal_space = {
r"[^\t\x20\xA0\x{1680}\x{180E}\x{2000}-\x{200A}\x{202F}\x{205F}\x{3000}]",
true,
},
horizontal_space = {
r"[\t\x20\xA0\x{1680}\x{180E}\x{2000}-\x{200A}\x{202F}\x{205F}\x{3000}]",
false,
},
non_vertical_space = { r"[^\n-\r\x{85}\x{2028}\x{2029}]", true },
vertical_space = { r"[\n-\r\x{85}\x{2028}\x{2029}]", false },
non_letter = { r"\P{L}", true },
letter = { r"\p{L}", false },
other_letter = { r"\p{Lo}", false },
other_symbol = { r"\p{So}", false },
surrogate = { r"\p{Cs}", true },
not_surrogate = { r"\P{Cs}", false },
other = { r"\p{C}", true },
not_other = { r"\P{C}", false },
unassigned = { r"\p{Cn}", false },
private_use = { r"\p{Co}", false },
control = { r"\p{IsControl}", false },
assigned = { r"\p{IsAssigned}", true },
not_assigned = { r"\P{IsAssigned}", false },
common_script = { r"\p{IsCommon}", false },
latin_script = { r"\p{IsLatin}", false },
alphabetic = { r"\p{IsAlphabetic}", false },
unicode_alpha = { r"[\p{Alphabetic}]", false },
white_space = { r"\p{IsWhite_Space}", false },
punctuation = { r"\p{IsPunctuation}", false },
emoji = { r"\p{IsEmoji}", false },
ascii = { r"\p{ASCII}", false },
not_ascii = { r"\P{ASCII}", true },
range_across_the_surrogates = { r"[\x{D000}-\x{E000}]", true },
negated_range_across_the_surrogates = { r"[^\x{D000}-\x{E000}]", false },
range_past_the_surrogates = { r"[\x{E000}-\x{F000}]", false },
negated_supplementary = { r"[^\x{1F600}]", true },
supplementary = { r"[\x{1F600}]", false },
supplementary_escape = { r"\x{1F600}", false },
intersection = { r"[[[a]&&[[^b]]]]", false },
negated_intersection = { r"[^[[a]&&[[^b]]]]", true },
negated_range_intersection = { r"[^[[a-z]&&[[^m]]]]", true },
intersection_with_surrogates = { r"[[[\p{L}]&&[\p{Cs}]]]", false },
surrogates_in_a_class = { r"[[[\P{L}]&&[\p{Cs}]]]", true },
intersection_of_negations = { r"[[[\P{L}]&&[[^a]]]]", true },
negated_union = { r"[^\p{L}\p{N}]", true },
union_with_a_negation = { r"[[^0-9A-Za-z_]0-9]", true },
upper_under_a_case_flag = { r"[\p{Lu}\p{Ll}\p{Lt}]", false },
)]
fn a_class_matches_a_lone_low_surrogate_as_javas(class: &str, expected: bool) {
let program = windowed(&format!("(?<={class})x"), 1, &[units(1, 1)]);
let marked = if expected { "😀<x>" } else { "😀x" };
check!(mark_matches(&program, "😀x") == marked, "{class:?}");
}
#[yare::parameterized(
surrogate_by_category = { r"\p{gc=Cs}", true },
surrogate_by_long_category = { r"\p{general_category=Cs}", true },
unknown_script = { r"\p{IsUnknown}", true },
unknown_script_code = { r"\p{IsZzzz}", true },
unknown_script_by_property = { r"\p{sc=Zzzz}", true },
unknown_script_by_long_property = { r"\p{script=Unknown}", true },
)]
fn a_class_the_regex_crate_refuses_is_read_for_surrogates(class: &str, expected: bool) {
let tree = Expr::parse_tree(class).unwrap_or_else(|_| panic!("{class:?} parses"));
let Expr::Delegate { inner, .. } = tree.expr else {
panic!("{class:?} is not a class");
};
check!(Class::new(&inner).is_none(), "{inner:?}");
check!(class_has_surrogates(&inner) == expected, "{inner:?}");
}
#[yare::parameterized(
group_with_a_flag = { r"(?i:[^a])", true },
flag_before = { r"(?i)[^a]", true },
perl_digit = { r"\d", false },
perl_non_space = { r"\S", true },
one_letter = { r"\pC", true },
not_equal = { r"\p{gc!=Cs}", false },
negated_not_equal = { r"\P{gc!=Cs}", true },
script_extensions = { r"\p{scx=Zzzz}", true },
fancy_not_surrogate = { r"\p{any}", false },
fancy_surrogate = { r"\P{any}", true },
difference = { r"[\p{Cs}--a]", true },
difference_of_all = { r"[\p{Cs}--\p{C}]", false },
symmetric_difference = { r"[\p{Cs}~~a]", true },
symmetric_difference_of_both = { r"[\p{Cs}~~\p{C}]", false },
ascii_class = { r"[[:alpha:]]", false },
negated_ascii_class = { r"[[:^alpha:]]", true },
two_classes = { "[^a][^b]", false },
alternation = { "[^a]|b", false },
unparsed = { "[^a", false },
)]
fn a_class_text_is_read_for_surrogates_by_its_parts(class: &str, expected: bool) {
check!(class_has_surrogates(class) == expected, "{class:?}");
}
}