use std::collections::HashMap;
use std::sync::Arc;
use super::Rune;
use super::charset::CharSet;
use super::parse::{Node, NodeKind, UNBOUNDED};
use super::units::RUNE_ERROR;
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(crate) enum Op {
Char,
CharBack,
Set,
SetBack,
Lit,
Fail,
Bol,
Eol,
WordBoundary,
NotWordBoundary,
LineStart,
LineEnd,
WordBoundaryFold,
NotWordBoundaryFold,
Jmp,
Split,
SaveTmp,
Capture,
Clear,
Backref,
BackrefBack,
BackrefFold,
BackrefFoldBack,
BackrefUnits,
BackrefUnitsBack,
Look,
LookOk,
LookFail,
Rep,
RepLazy,
LoopInit,
Loop,
LoopIter,
Match,
}
#[derive(Clone, Debug)]
pub(crate) struct Inst {
pub op: Op,
pub flag: bool,
pub r: Rune,
pub set: Option<Arc<CharSet>>,
pub lit: Box<[u8]>,
pub x: i32,
pub y: i32,
pub a: i32,
pub b: i32,
pub min: i32,
pub max: i32,
pub need: i32,
pub iter: i32,
pub m: i32,
}
impl Inst {
fn new(op: Op) -> Inst {
Inst {
op,
flag: false,
r: 0,
set: None,
lit: Box::default(),
x: 0,
y: 0,
a: 0,
b: 0,
min: 0,
max: 0,
need: 0,
iter: 0,
m: 0,
}
}
}
pub(crate) const MAX_MEMO_LOOPS: usize = 2;
#[derive(Clone, Copy, Debug)]
pub(crate) struct MemoPoint {
pub loops: [i32; MAX_MEMO_LOOPS],
pub mins: [i32; MAX_MEMO_LOOPS],
}
#[derive(Clone, Copy)]
struct LoopContext {
loop_: i32,
min: i32,
simple: bool,
}
#[derive(Debug)]
pub(crate) struct Program {
pub insts: Vec<Inst>,
pub groups: usize,
pub loops: usize,
pub units: bool,
pub anchored: bool,
pub prefix: String,
pub need: usize,
pub points: Vec<MemoPoint>,
pub quick: Vec<Quick>,
pub first: Option<CharSet>,
}
struct Compiler {
insts: Vec<Inst>,
tracked: HashMap<i32, i32>,
loops: usize,
unicode: bool,
memo: bool,
open: Vec<LoopContext>,
points: Vec<MemoPoint>,
}
pub(crate) fn compile_program(root: Node, unicode: bool) -> Program {
let mut open = Vec::new();
let root = simplify(root, &mut open);
let mut c = Compiler {
insts: Vec::new(),
tracked: HashMap::new(),
loops: 0,
unicode,
memo: false,
open: Vec::new(),
points: Vec::new(),
};
let mut referenced = Vec::new();
collect_backrefs(&root, &mut referenced);
referenced.sort_unstable();
referenced.dedup();
for (slot, g) in referenced.into_iter().enumerate() {
c.tracked.insert(g, slot as i32);
}
c.memo = c.tracked.is_empty();
c.emit(&root, false);
c.add(Inst::new(Op::Match));
let mut insts = c.insts;
let need = set_needs(&mut insts);
let quick = set_quick(&insts);
let mut p = Program {
insts,
groups: c.tracked.len(),
loops: c.loops,
units: !unicode,
anchored: starts_at_bol(&root),
prefix: String::new(),
need,
points: c.points,
quick,
first: None,
};
if !p.anchored {
literal_prefix(&root, &mut p.prefix);
if p.prefix.is_empty() {
let mut pairs = Vec::new();
if let Some(nullable) = first_chars(&root, &mut pairs)
&& !nullable
{
p.first = Some(CharSet::new(&pairs, false));
}
}
}
p
}
pub(crate) fn simplify(mut n: Node, open: &mut Vec<i32>) -> Node {
if n.kind == NodeKind::Backref {
if n.groups.iter().all(|g| open.contains(g)) {
return Node::new(NodeKind::Empty);
}
return n;
}
if n.subs.is_empty() {
return n;
}
let opened = n.kind == NodeKind::Group && n.group > 0;
if opened {
open.push(n.group);
}
n.subs = std::mem::take(&mut n.subs).into_iter().map(|sub| simplify(sub, open)).collect();
if opened {
open.pop();
}
if n.kind == NodeKind::Repeat && n.max != 0 && !consumes(&n.subs[0]) {
if n.min == 0 {
return Node::new(NodeKind::Empty);
}
return n.subs.pop().expect("a repetition has a body");
}
n
}
fn consumes(n: &Node) -> bool {
match n.kind {
NodeKind::Char | NodeKind::Set | NodeKind::Backref => true,
NodeKind::Cat | NodeKind::Alt | NodeKind::Group => n.subs.iter().any(consumes),
NodeKind::Repeat => n.max != 0 && consumes(&n.subs[0]),
_ => false,
}
}
fn collect_backrefs(n: &Node, out: &mut Vec<i32>) {
if n.kind == NodeKind::Backref {
out.extend_from_slice(&n.groups);
}
for sub in &n.subs {
collect_backrefs(sub, out);
}
}
fn push_rune(out: &mut String, r: Rune) {
out.push(char::from_u32(r as u32).expect("a literal code point is a character"));
}
impl Compiler {
fn add(&mut self, i: Inst) -> i32 {
self.insts.push(i);
(self.insts.len() - 1) as i32
}
fn pc(&self) -> i32 {
self.insts.len() as i32
}
fn point(&mut self) -> i32 {
if !self.memo || self.open.len() > MAX_MEMO_LOOPS {
return 0;
}
let mut p = MemoPoint { loops: [-1; MAX_MEMO_LOOPS], mins: [0; MAX_MEMO_LOOPS] };
for (k, l) in self.open.iter().enumerate() {
if !l.simple {
return 0;
}
(p.loops[k], p.mins[k]) = (l.loop_, l.min);
}
self.points.push(p);
self.points.len() as i32
}
fn group_range(&self, n: &Node) -> Option<(i32, i32)> {
fn walk(c: &Compiler, n: &Node, range: &mut Option<(i32, i32)>) {
if n.kind == NodeKind::Group
&& n.group > 0
&& let Some(&t) = c.tracked.get(&n.group)
{
*range = Some(match *range {
Some((lo, hi)) => (lo.min(t), hi.max(t)),
None => (t, t),
});
}
for sub in &n.subs {
walk(c, sub, range);
}
}
let mut range = None;
walk(self, n, &mut range);
range
}
fn emit(&mut self, n: &Node, back: bool) {
match n.kind {
NodeKind::Empty => {}
NodeKind::Char => {
self.add(Inst { r: n.r, ..Inst::new(if back { Op::CharBack } else { Op::Char }) });
}
NodeKind::Set => {
let set = n.class();
if set.is_empty() {
self.add(Inst::new(Op::Fail));
} else {
self.add(Inst { set: Some(set.clone()), ..Inst::new(if back { Op::SetBack } else { Op::Set }) });
}
}
NodeKind::Cat => {
if back {
for sub in n.subs.iter().rev() {
self.emit(sub, true);
}
return;
}
let mut i = 0;
while i < n.subs.len() {
let mut j = i;
let mut lit = String::new();
while j < n.subs.len() && n.subs[j].kind == NodeKind::Char && literal_rune(n.subs[j].r) {
push_rune(&mut lit, n.subs[j].r);
j += 1;
}
if j - i >= 2 {
self.add(Inst { lit: lit.into_bytes().into_boxed_slice(), ..Inst::new(Op::Lit) });
i = j;
continue;
}
self.emit(&n.subs[i], false);
i += 1;
}
}
NodeKind::Alt => {
let mut jumps = Vec::new();
for (i, sub) in n.subs.iter().enumerate() {
if i == n.subs.len() - 1 {
self.emit(sub, back);
break;
}
let m = self.point();
let split = self.add(Inst { m, ..Inst::new(Op::Split) });
self.insts[split as usize].x = self.pc();
self.emit(sub, back);
jumps.push(self.add(Inst::new(Op::Jmp)));
self.insts[split as usize].y = self.pc();
}
for j in jumps {
self.insts[j as usize].x = self.pc();
}
}
NodeKind::Group => {
let tracked = if n.group == 0 { None } else { self.tracked.get(&n.group).copied() };
let Some(t) = tracked else {
self.emit(&n.subs[0], back);
return;
};
self.add(Inst { a: t, ..Inst::new(Op::SaveTmp) });
self.emit(&n.subs[0], back);
self.add(Inst { a: t, flag: back, ..Inst::new(Op::Capture) });
}
NodeKind::Bol => {
self.add(Inst::new(Op::Bol));
}
NodeKind::Eol => {
self.add(Inst::new(Op::Eol));
}
NodeKind::LineStart => {
self.add(Inst::new(Op::LineStart));
}
NodeKind::LineEnd => {
self.add(Inst::new(Op::LineEnd));
}
NodeKind::WordBoundary => {
self.add(Inst::new(if n.fold { Op::WordBoundaryFold } else { Op::WordBoundary }));
}
NodeKind::NotWordBoundary => {
self.add(Inst::new(if n.fold { Op::NotWordBoundaryFold } else { Op::NotWordBoundary }));
}
NodeKind::Backref => {
let op = match (n.fold, back, self.unicode) {
(true, true, _) => Op::BackrefFoldBack,
(true, false, _) => Op::BackrefFold,
(false, true, false) => Op::BackrefUnitsBack,
(false, true, true) => Op::BackrefBack,
(false, false, false) => Op::BackrefUnits,
(false, false, true) => Op::Backref,
};
for g in &n.groups {
self.add(Inst { a: self.tracked[g], flag: self.unicode, ..Inst::new(op) });
}
}
NodeKind::Look => {
let look = self.add(Inst { flag: n.negative, ..Inst::new(Op::Look) });
let outer = std::mem::take(&mut self.open);
self.emit(&n.subs[0], n.behind);
self.open = outer;
self.add(Inst::new(if n.negative { Op::LookFail } else { Op::LookOk }));
self.insts[look as usize].x = self.pc();
}
NodeKind::Repeat => self.emit_repeat(n, back),
}
}
fn emit_body(&mut self, body: &Node, back: bool, clear: Option<(i32, i32)>) {
if let Some((lo, hi)) = clear {
self.add(Inst { a: lo, b: hi, ..Inst::new(Op::Clear) });
}
self.emit(body, back);
}
fn split(&mut self) -> i32 {
let m = self.point();
self.add(Inst { m, ..Inst::new(Op::Split) })
}
fn set_split(&mut self, lazy: bool, at: i32, enter: i32, exit: i32) {
let i = &mut self.insts[at as usize];
if lazy {
(i.x, i.y) = (exit, enter);
} else {
(i.x, i.y) = (enter, exit);
}
}
fn emit_repeat(&mut self, n: &Node, back: bool) {
let body = &n.subs[0];
if n.max == 0 {
return;
}
let (min, max) = (n.min, n.max);
if !back && (body.kind == NodeKind::Char || body.kind == NodeKind::Set && !body.class().is_empty()) {
let m = self.point();
self.add(Inst {
r: body.r,
set: body.set.clone(),
min,
max,
m,
..Inst::new(if n.lazy { Op::RepLazy } else { Op::Rep })
});
return;
}
let clear = self.group_range(body);
if !nullable(body) {
if n.min == 0 && n.max == UNBOUNDED {
let at = self.split();
let enter = self.pc();
self.emit_body(body, back, clear);
self.add(Inst { x: at, ..Inst::new(Op::Jmp) });
let exit = self.pc();
self.set_split(n.lazy, at, enter, exit);
return;
}
if n.min == 1 && n.max == UNBOUNDED {
let enter = self.pc();
self.emit_body(body, back, clear);
let at = self.split();
let exit = self.pc();
self.set_split(n.lazy, at, enter, exit);
return;
}
if n.min == 0 && n.max == 1 {
let at = self.split();
let enter = self.pc();
self.emit_body(body, back, clear);
let exit = self.pc();
self.set_split(n.lazy, at, enter, exit);
return;
}
}
let k = self.loops as i32;
self.loops += 1;
self.add(Inst { a: k, ..Inst::new(Op::LoopInit) });
let iter = if back { 0 } else { min_length(body) };
let simple = max == UNBOUNDED && min <= 1;
let point = if simple { self.point() } else { 0 };
let loop_ = self.add(Inst { a: k, min, max, flag: n.lazy, iter: iter as i32, m: point, ..Inst::new(Op::Loop) });
self.add(Inst { a: k, ..Inst::new(Op::LoopIter) });
self.open.push(LoopContext { loop_: k, min, simple });
self.emit_body(body, back, clear);
self.open.pop();
self.add(Inst { x: loop_, ..Inst::new(Op::Jmp) });
self.insts[loop_ as usize].x = self.pc();
}
}
pub(crate) const NO_MATCH: i64 = 1 << 30;
fn rune_len(r: Rune) -> i64 {
match r {
0..=0x7F => 1,
0x80..=0x7FF => 2,
0x800..=0xFFFF => 3,
_ => 4,
}
}
fn min_length(n: &Node) -> i64 {
match n.kind {
NodeKind::Char => {
if literal_rune(n.r) {
rune_len(n.r)
} else {
1
}
}
NodeKind::Set => {
if n.class().is_empty() {
NO_MATCH
} else {
1
}
}
NodeKind::Cat => {
let mut total = 0;
for sub in &n.subs {
total += min_length(sub);
if total >= NO_MATCH {
return NO_MATCH;
}
}
total
}
NodeKind::Alt => n.subs.iter().map(min_length).min().unwrap_or(NO_MATCH).min(NO_MATCH),
NodeKind::Group => min_length(&n.subs[0]),
NodeKind::Repeat => times_need(n.min as i64, min_length(&n.subs[0])),
_ => 0,
}
}
fn times_need(count: i64, need: i64) -> i64 {
if count == 0 || need == 0 {
return 0;
}
if need >= NO_MATCH || count >= NO_MATCH / need {
return NO_MATCH;
}
count * need
}
pub(crate) fn set_needs(insts: &mut [Inst]) -> usize {
needs(insts).0
}
pub(crate) fn needs(insts: &mut [Inst]) -> (usize, usize) {
let mut need = vec![NO_MATCH as i32; insts.len()];
let mut passes = 0;
let mut changed = true;
while changed {
changed = false;
passes += 1;
for pc in (0..insts.len()).rev() {
let i = &insts[pc];
let mut n = NO_MATCH;
let mut next = |to: i32, weight: i64| {
let v = need[to as usize] as i64 + weight;
if v < n {
n = v;
}
};
let after = (pc + 1) as i32;
match i.op {
Op::Match | Op::LookOk | Op::LookFail => n = 0,
Op::Fail => {}
Op::Char | Op::Set => next(after, 1),
Op::Lit => next(after, i.lit.len() as i64),
Op::Rep | Op::RepLazy => next(after, i.min as i64),
Op::Jmp => {
let head = &insts[i.x as usize];
if (i.x as usize) < pc && head.op == Op::Loop {
next(head.x, 0);
} else if (i.x as usize) < pc && head.op == Op::Split {
next(head.x, 0);
next(head.y, 0);
} else {
next(i.x, 0);
}
}
Op::Look => next(i.x, 0),
Op::Split => {
next(i.x, 0);
next(i.y, 0);
}
Op::Loop => {
next(after, 0);
next(i.x, 0);
}
Op::LoopInit => {
let loop_ = &insts[pc + 1];
next(after, times_need(loop_.min as i64, loop_.iter as i64));
}
_ => next(after, 0),
}
let n = n.min(NO_MATCH) as i32;
if n < need[pc] {
need[pc] = n;
changed = true;
}
}
}
for (pc, i) in insts.iter_mut().enumerate() {
if i.op == Op::Loop {
i.need = need[pc];
}
}
(need[0] as usize, passes)
}
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub(crate) struct Quick {
pub ascii: [u64; 2],
pub other: bool,
pub mid: bool,
pub end: bool,
}
impl Quick {
#[inline(always)]
pub(crate) fn allows(&self, input: &[u8], pos: usize) -> bool {
if pos >= input.len() {
return self.end;
}
if self.mid {
return true;
}
let c = input[pos];
if c < 0x80 {
return (self.ascii[(c >> 6) as usize] >> (c & 63)) & 1 != 0;
}
self.other
}
fn add_rune(&mut self, r: Rune) {
if (0..0x80).contains(&r) {
self.ascii[(r >> 6) as usize] |= 1 << (r & 63);
} else {
self.other = true;
}
}
fn add_set(&mut self, s: &CharSet) {
self.ascii[0] |= s.ascii[0];
self.ascii[1] |= s.ascii[1];
if s.ranges.last().is_some_and(|&last| last >= 0x80) {
self.other = true;
}
}
fn union(&mut self, o: Quick) {
self.ascii[0] |= o.ascii[0];
self.ascii[1] |= o.ascii[1];
self.other = self.other || o.other;
self.mid = self.mid || o.mid;
self.end = self.end || o.end;
}
}
pub(crate) fn set_quick(insts: &[Inst]) -> Vec<Quick> {
quicks(insts).0
}
pub(crate) fn quicks(insts: &[Inst]) -> (Vec<Quick>, usize) {
let mut q = vec![Quick::default(); insts.len()];
let all = Quick { ascii: [!0, !0], other: true, mid: true, end: true };
let mut passes = 0;
let mut changed = true;
while changed {
changed = false;
passes += 1;
for pc in (0..insts.len()).rev() {
let i = &insts[pc];
let mut n = Quick::default();
match i.op {
Op::Match | Op::LookOk | Op::LookFail => n = all,
Op::Fail => {}
Op::Char => n.add_rune(i.r),
Op::Set => n.add_set(i.set.as_ref().expect("a set instruction has a class")),
Op::Lit => n.add_rune(i.lit[0] as Rune),
Op::Rep | Op::RepLazy => {
match &i.set {
Some(set) => n.add_set(set),
None => n.add_rune(i.r),
}
if i.min == 0 {
n.union(q[pc + 1]);
}
}
Op::Jmp => {
n = q[i.x as usize];
let head = &insts[i.x as usize];
if (i.x as usize) < pc && head.op == Op::Loop {
n.union(q[head.x as usize]);
} else if (i.x as usize) < pc && head.op == Op::Split {
n.union(q[head.x as usize]);
n.union(q[head.y as usize]);
}
}
Op::Split => {
n = q[i.x as usize];
n.union(q[i.y as usize]);
}
Op::Loop => {
n = q[pc + 1];
n.union(q[i.x as usize]);
}
Op::Eol => n.end = q[pc + 1].end,
Op::Look => {
n = q[i.x as usize];
if !i.flag {
let b = q[pc + 1];
n.ascii[0] &= b.ascii[0];
n.ascii[1] &= b.ascii[1];
(n.other, n.mid, n.end) = (n.other && b.other, n.mid && b.mid, n.end && b.end);
}
}
Op::CharBack
| Op::SetBack
| Op::Backref
| Op::BackrefBack
| Op::BackrefFold
| Op::BackrefFoldBack
| Op::BackrefUnits
| Op::BackrefUnitsBack => n = all,
_ => n = q[pc + 1],
}
if n != q[pc] {
q[pc] = n;
changed = true;
}
}
}
(q, passes)
}
pub(crate) fn literal_rune(r: Rune) -> bool {
(0..=0x10FFFF).contains(&r) && !(0xD800..=0xDFFF).contains(&r) && r != RUNE_ERROR
}
fn nullable(n: &Node) -> bool {
match n.kind {
NodeKind::Char | NodeKind::Set => false,
NodeKind::Cat => n.subs.iter().all(nullable),
NodeKind::Alt => n.subs.iter().any(nullable),
NodeKind::Group => nullable(&n.subs[0]),
NodeKind::Repeat => n.min == 0 || nullable(&n.subs[0]),
_ => true,
}
}
fn starts_at_bol(n: &Node) -> bool {
match n.kind {
NodeKind::Bol => true,
NodeKind::Cat => starts_at_bol(&n.subs[0]),
NodeKind::Alt => n.subs.iter().all(starts_at_bol),
NodeKind::Group => starts_at_bol(&n.subs[0]),
NodeKind::Repeat => n.min >= 1 && starts_at_bol(&n.subs[0]),
_ => false,
}
}
fn literal_prefix(n: &Node, prefix: &mut String) {
match n.kind {
NodeKind::Char => {
if literal_rune(n.r) {
push_rune(prefix, n.r);
}
}
NodeKind::Cat => {
for sub in &n.subs {
if sub.kind != NodeKind::Char || !literal_rune(sub.r) {
if prefix.is_empty() && sub.kind == NodeKind::Group {
literal_prefix(sub, prefix);
}
break;
}
push_rune(prefix, sub.r);
}
}
NodeKind::Group => literal_prefix(&n.subs[0], prefix),
_ => {}
}
}
fn first_chars(n: &Node, pairs: &mut Vec<Rune>) -> Option<bool> {
match n.kind {
NodeKind::Char => {
pairs.push(n.r);
pairs.push(n.r);
Some(false)
}
NodeKind::Set => {
pairs.extend_from_slice(&n.class().ranges);
Some(false)
}
NodeKind::Cat => {
for sub in &n.subs {
if !first_chars(sub, pairs)? {
return Some(false);
}
}
Some(true)
}
NodeKind::Alt => {
let mut empty = false;
for sub in &n.subs {
empty |= first_chars(sub, pairs)?;
}
Some(empty)
}
NodeKind::Group => first_chars(&n.subs[0], pairs),
NodeKind::Repeat => {
if n.max == 0 {
return Some(true);
}
let empty = first_chars(&n.subs[0], pairs)?;
Some(empty || n.min == 0)
}
NodeKind::Backref => None,
_ => Some(true),
}
}