use super::AnchorContext;
use super::ast::{AnchorKind, Ast, ClassAtom, LookKind, ParsedRegex, PerlClassKind};
use super::backtrack::{
StartByteSet, class_start_bytes, expand_case_insensitive_start_bytes,
is_cpp_space_comment_separator, is_perl_class, strip_nonsemantic_group,
};
use std::sync::Arc;
const ASCII_WHITESPACE: [u8; 6] = [b' ', b'\t', b'\n', b'\r', 0x0b, 0x0c];
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) struct SkipGate(Arc<SkipGateParts>);
#[derive(Debug, PartialEq, Eq)]
struct SkipGateParts {
start: Option<StartAssert>,
rest: Option<RestGate>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
struct StartAssert {
line_start: bool,
continuation: bool,
prev_bytes: StartByteSet,
cur_bytes: StartByteSet,
}
impl StartAssert {
fn never() -> Self {
Self {
line_start: false,
continuation: false,
prev_bytes: StartByteSet::empty(),
cur_bytes: StartByteSet::empty(),
}
}
fn line_start() -> Self {
Self {
line_start: true,
..Self::never()
}
}
fn continuation() -> Self {
Self {
continuation: true,
..Self::never()
}
}
fn prev_bytes(prev_bytes: StartByteSet) -> Self {
Self {
prev_bytes,
..Self::never()
}
}
fn cur_bytes(cur_bytes: StartByteSet) -> Self {
Self {
cur_bytes,
..Self::never()
}
}
fn only_cur_bytes(&self) -> bool {
!self.line_start && !self.continuation && self.prev_bytes.is_empty()
}
fn union(left: Option<Self>, right: Option<Self>) -> Option<Self> {
let (mut left, right) = (left?, right?);
left.line_start |= right.line_start;
left.continuation |= right.continuation;
left.prev_bytes.extend(&right.prev_bytes);
left.cur_bytes.extend(&right.cur_bytes);
Some(left)
}
fn allows(&self, bytes: &[u8], start: usize, ctx: AnchorContext) -> bool {
(self.line_start && start == 0)
|| (self.continuation && ctx.allow_g && ctx.g_pos == start)
|| start
.checked_sub(1)
.and_then(|prev| bytes.get(prev))
.is_some_and(|byte| self.prev_bytes.contains(*byte))
|| bytes
.get(start)
.is_some_and(|byte| self.cur_bytes.contains(*byte))
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
struct RestGate {
rest_bytes: StartByteSet,
allow_empty: bool,
allow_whitespace: bool,
allow_comment: bool,
}
#[derive(Default)]
pub(crate) struct SkipGateLineState {
whitespace_run: Option<(usize, usize)>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum SkipGateDecision {
Allow,
Skip,
NeedsCommentCheck,
}
impl SkipGate {
#[cfg(test)]
pub(crate) fn analyze(parsed: &ParsedRegex) -> Option<Self> {
parsed.analysis().skip_gate().cloned()
}
pub(crate) fn analyze_with_effective_flags(
parsed: &ParsedRegex,
uniform_flags: Option<super::ast::RegexFlags>,
has_case_insensitive_scope: bool,
start_restricted: bool,
) -> Option<Self> {
let mut walk = PrefixWalk {
bytes: StartByteSet::empty(),
unbounded: false,
allow_empty: false,
allow_whitespace: false,
allow_comment: false,
stopped: None,
case_folding: has_case_insensitive_scope
|| parsed.flags.case_insensitive
|| uniform_flags.is_some_and(|flags| flags.case_insensitive),
};
let continuing = walk.visit(&parsed.ast, PathState::START);
let all_stopped = continuing.is_none();
let start = match (walk.stopped.take(), continuing) {
(Some(stopped), Some(path)) => StartAssert::union(stopped, path.assert),
(Some(stopped), None) => stopped,
(None, Some(path)) => path.assert,
(None, None) => None,
}
.filter(|start| !(start_restricted && start.only_cur_bytes()));
let rest = if all_stopped {
walk.rest_gate(parsed, uniform_flags, has_case_insensitive_scope)
} else {
None
};
(start.is_some() || rest.is_some()).then(|| Self(Arc::new(SkipGateParts { start, rest })))
}
pub(crate) fn decide(
&self,
line: &str,
start: usize,
ctx: AnchorContext,
state: &mut SkipGateLineState,
) -> SkipGateDecision {
if let Some(assert) = &self.0.start
&& !assert.allows(line.as_bytes(), start, ctx)
{
return SkipGateDecision::Skip;
}
match &self.0.rest {
Some(rest) => rest.decide(line, start, state),
None => SkipGateDecision::Allow,
}
}
}
impl RestGate {
fn decide(&self, line: &str, start: usize, state: &mut SkipGateLineState) -> SkipGateDecision {
let bytes = line.as_bytes();
if self.allow_empty
&& bytes
.get(start)
.is_some_and(|byte| self.rest_bytes.contains(*byte))
{
return SkipGateDecision::Allow;
}
if self.allow_whitespace {
match whitespace_run_end(bytes, start, &mut state.whitespace_run) {
Some(end) => {
if end > start
&& bytes
.get(end)
.is_some_and(|byte| self.rest_bytes.contains(*byte))
{
return SkipGateDecision::Allow;
}
}
None => return SkipGateDecision::Allow,
}
}
if self.allow_comment {
SkipGateDecision::NeedsCommentCheck
} else {
SkipGateDecision::Skip
}
}
}
#[derive(Debug, Clone)]
struct PathState {
at_start: bool,
pristine: bool,
assert: Option<StartAssert>,
}
impl PathState {
const START: Self = Self {
at_start: true,
pristine: true,
assert: None,
};
fn merge(left: Option<Self>, right: Option<Self>) -> Option<Self> {
match (left, right) {
(Some(left), Some(right)) => Some(Self {
at_start: left.at_start || right.at_start,
pristine: left.pristine && right.pristine,
assert: StartAssert::union(left.assert, right.assert),
}),
(path, None) | (None, path) => path,
}
}
fn assume(mut self, assert: StartAssert) -> Self {
if self.assert.is_none() {
self.assert = Some(assert);
}
self
}
fn consumed(self, nullable: bool) -> Self {
Self {
at_start: self.at_start && nullable,
pristine: false,
..self
}
}
}
#[derive(Clone)]
struct PrefixWalk {
bytes: StartByteSet,
unbounded: bool,
allow_empty: bool,
allow_whitespace: bool,
allow_comment: bool,
stopped: Option<Option<StartAssert>>,
case_folding: bool,
}
impl PrefixWalk {
fn token(&mut self, path: PathState, bytes: Option<&StartByteSet>) -> Option<PathState> {
match bytes {
Some(bytes) => self.bytes.extend(bytes),
None => self.unbounded = true,
}
self.allow_empty |= path.at_start;
let mut assert = path.assert;
if assert.is_none()
&& path.pristine
&& !self.case_folding
&& let Some(bytes) = bytes
{
assert = Some(StartAssert::cur_bytes(bytes.clone()));
}
self.stopped = Some(match self.stopped.take() {
Some(stopped) => StartAssert::union(stopped, assert),
None => assert,
});
None
}
fn visit(&mut self, ast: &Ast, path: PathState) -> Option<PathState> {
match classify_skip_element(ast) {
Some(SkipElement::Separator) => {
self.allow_whitespace = true;
self.allow_comment = true;
return Some(path.consumed(true));
}
Some(SkipElement::Whitespace { nullable }) => {
self.allow_whitespace = true;
return Some(path.consumed(nullable));
}
None => {}
}
match ast {
Ast::Empty => Some(path),
Ast::Anchor(AnchorKind::LineStart | AnchorKind::TextStart) => {
Some(path.assume(StartAssert::line_start()))
}
Ast::Anchor(AnchorKind::Continuation) if path.pristine => {
Some(path.assume(StartAssert::continuation()))
}
Ast::Anchor(_) => Some(path),
Ast::Look {
kind: LookKind::Behind,
child,
} if path.pristine && !self.case_folding => match behind_assert(child) {
Some(assert) => Some(path.assume(assert)),
None => Some(path),
},
Ast::Look {
kind: LookKind::Ahead,
child,
} => {
let mut inner = self.clone();
if inner.visit(child, path.clone()).is_none()
&& (self.unbounded || !inner.unbounded)
{
*self = inner;
None
} else {
Some(path)
}
}
Ast::Look { .. } => Some(path),
Ast::Literal(literal) => match literal.chars().next() {
None => Some(path),
Some(ch) if ch.is_ascii() => {
let mut bytes = StartByteSet::empty();
bytes.insert(ch as u8);
self.token(path, Some(&bytes))
}
Some(_) => self.token(path, None),
},
Ast::Class(class) => self.token(path, class_start_bytes(class).as_ref()),
Ast::Dot | Ast::Grapheme => self.token(path, None),
Ast::Concat(nodes) => {
let mut path = path;
for node in nodes {
path = self.visit(node, path)?;
}
Some(path)
}
Ast::Alternation(branches) => {
let mut merged = None;
for branch in branches {
let branch = self.visit(branch, path.clone());
merged = PathState::merge(merged, branch);
}
merged
}
Ast::Repeat { node, min, max, .. } => {
if *max == Some(0) {
return Some(path);
}
let through = self.visit(node, path.clone());
if *min == 0 {
PathState::merge(through, Some(path))
} else {
through
}
}
Ast::Group { child, .. } | Ast::Flags { child, .. } => self.visit(child, path),
Ast::Backref(_)
| Ast::Conditional { .. }
| Ast::Subroutine(_)
| Ast::Unsupported(_) => {
self.unbounded = true;
Some(path.consumed(true))
}
}
}
fn rest_gate(
self,
parsed: &ParsedRegex,
uniform_flags: Option<super::ast::RegexFlags>,
has_case_insensitive_scope: bool,
) -> Option<RestGate> {
if self.unbounded || self.bytes.is_empty() || !(self.allow_whitespace || self.allow_comment)
{
return None;
}
if has_case_insensitive_scope && uniform_flags.is_none() {
return None;
}
let mut rest_bytes = self.bytes;
if uniform_flags.unwrap_or(parsed.flags).case_insensitive {
expand_case_insensitive_start_bytes(&mut rest_bytes);
}
if ASCII_WHITESPACE
.iter()
.any(|byte| rest_bytes.contains(*byte))
{
return None;
}
Some(RestGate {
rest_bytes,
allow_empty: self.allow_empty,
allow_whitespace: self.allow_whitespace,
allow_comment: self.allow_comment,
})
}
}
fn behind_assert(child: &Ast) -> Option<StartAssert> {
match child {
Ast::Anchor(AnchorKind::LineStart | AnchorKind::TextStart) => {
Some(StartAssert::line_start())
}
Ast::Group { child, .. } => behind_assert(child),
Ast::Alternation(branches) => branches
.iter()
.map(behind_assert)
.reduce(StartAssert::union)
.flatten(),
Ast::Concat(nodes) => behind_assert(nodes.last()?),
Ast::Repeat { node, min, .. } if *min >= 1 => behind_assert(node),
Ast::Literal(literal) => {
let last = literal.chars().next_back()?;
last.is_ascii().then(|| {
let mut bytes = StartByteSet::empty();
bytes.insert(last as u8);
StartAssert::prev_bytes(bytes)
})
}
Ast::Class(class) => {
let bytes = class_start_bytes(class)?;
(0x80..=0xff)
.all(|byte| !bytes.contains(byte))
.then(|| StartAssert::prev_bytes(bytes))
}
_ => None,
}
}
enum SkipElement {
Separator,
Whitespace { nullable: bool },
}
fn classify_skip_element(ast: &Ast) -> Option<SkipElement> {
let stripped = strip_flags(strip_nonsemantic_group(ast));
if let Ast::Alternation(branches) = stripped
&& is_cpp_space_comment_separator(branches)
{
return Some(SkipElement::Separator);
}
if is_perl_class(strip_flags(stripped), PerlClassKind::Space) {
return Some(SkipElement::Whitespace { nullable: false });
}
if let Ast::Repeat { node, min, max, .. } = stripped
&& max.is_none_or(|max| max >= *min)
&& is_perl_class(
strip_flags(strip_nonsemantic_group(node)),
PerlClassKind::Space,
)
{
return Some(SkipElement::Whitespace {
nullable: *min == 0,
});
}
None
}
fn strip_flags(ast: &Ast) -> &Ast {
let mut ast = ast;
loop {
match ast {
Ast::Flags { child, .. } => ast = strip_nonsemantic_group(child),
_ => return ast,
}
}
}
fn whitespace_run_end(
bytes: &[u8],
start: usize,
memo: &mut Option<(usize, usize)>,
) -> Option<usize> {
if let Some((memo_start, memo_end)) = *memo
&& start >= memo_start
&& start < memo_end
{
return Some(memo_end);
}
let mut end = start;
while let Some(byte) = bytes.get(end) {
if matches!(*byte, b'\t' | b'\n' | 0x0b | 0x0c | b'\r' | b' ') {
end += 1;
} else if !byte.is_ascii() {
return None;
} else {
break;
}
}
if end > start {
*memo = Some((start, end));
}
Some(end)
}
#[cfg(test)]
mod tests {
use super::super::ast::parse;
use super::*;
const SEPARATOR: &str =
r"((?:\s*+/\*(?:[^*]++|\*+(?!/))*+\*/\s*+)+|\s++|(?<=\W)|(?=\W)|^|\n?$|\A|\Z)";
fn gate(pattern: &str) -> Option<SkipGate> {
SkipGate::analyze(&parse(pattern))
}
fn allows(pattern: &str, line: &str, start: usize) -> bool {
match gate(pattern).expect("pattern should have a gate").decide(
line,
start,
AnchorContext::default(),
&mut SkipGateLineState::default(),
) {
SkipGateDecision::Allow => true,
SkipGateDecision::Skip => false,
SkipGateDecision::NeedsCommentCheck => {
memchr::memmem::find(line.as_bytes(), b"/*").is_some()
}
}
}
#[test]
fn separator_prefixed_keyword_gates_on_token_byte() {
let pattern = format!("{SEPARATOR}((?<!\\w)this(?!\\w))");
assert!(allows(&pattern, "this", 0));
assert!(allows(&pattern, " this", 0));
assert!(allows(&pattern, "x this", 1));
assert!(!allows(&pattern, " that_", 7));
assert!(!allows(&pattern, " #define", 0));
assert!(allows(&pattern, "/* c */ this", 0));
assert!(allows(&pattern, " /* c */ x", 0));
}
#[test]
fn whitespace_prefixed_type_set_gates_on_first_letters() {
let pattern = r"\s*+(?<!\w)(?:(unsigned|signed|double)(?!\w))";
assert!(allows(pattern, " unsigned x", 0));
assert!(allows(pattern, "signed", 0));
assert!(!allows(pattern, " (cast)", 0));
assert!(!allows(pattern, " 12345", 1));
}
#[test]
fn mandatory_whitespace_requires_the_run() {
let pattern = r"\s++(#)";
assert!(!allows(pattern, "#x", 0));
assert!(allows(pattern, " #x", 0));
}
#[test]
fn non_ascii_whitespace_disables_the_gate() {
let pattern = format!("{SEPARATOR}(#)");
assert!(allows(&pattern, " \u{a0} #", 0));
}
#[test]
fn patterns_without_skip_shape_have_no_gate() {
assert!(gate(r"[A-Za-z_]\w*").is_none());
assert!(gate(r"(?<!\w)this").is_none());
assert!(gate(r"\s*+\S+").is_none(), "rest may start with anything");
assert!(
gate(r"\s*+ ?#").is_none(),
"rest starting with whitespace defeats the run shortcut"
);
}
#[test]
fn optional_prefix_elements_contribute_token_bytes() {
let pattern = format!(r"(\s*+(\[\[.*?]])?{SEPARATOR}(?:unsigned|long)\b");
assert!(allows(&pattern, " unsigned x", 0));
assert!(allows(&pattern, "[[x]] long", 0));
assert!(allows(&pattern, " [[x]] long", 0));
assert!(!allows(&pattern, " (x)", 0));
assert!(!allows(&pattern, " signed", 1));
}
#[test]
fn leading_anchors_and_lookbehinds_gate_start_positions() {
let decide = |pattern: &str, line: &str, start: usize, ctx: AnchorContext| {
gate(pattern).expect("pattern should have a gate").decide(
line,
start,
ctx,
&mut SkipGateLineState::default(),
)
};
let line_start = AnchorContext::line_start();
let pattern = format!(r"^({SEPARATOR}(#)\s*define)\b");
assert_eq!(
decide(&pattern, "#define", 0, line_start),
SkipGateDecision::Allow
);
assert_eq!(
decide(&pattern, " #define", 1, line_start),
SkipGateDecision::Skip
);
let pattern = r"(?:(?:^|\G|(?<=[;}]))|(?<=>|\*/))\s*+\w+";
assert_eq!(
decide(pattern, "a;b", 2, line_start),
SkipGateDecision::Allow
);
assert_eq!(
decide(pattern, "a*/b", 3, line_start),
SkipGateDecision::Allow
);
assert_eq!(
decide(pattern, "a,b", 2, line_start),
SkipGateDecision::Skip
);
assert_eq!(
decide(pattern, "a,b", 2, AnchorContext::continuation(2)),
SkipGateDecision::Allow
);
assert!(gate(r"\s?(?<=[ \t])x").is_some_and(|gate| gate.0.start.is_none()));
assert!(gate(r"\s*^x").is_some_and(|gate| gate.0.start.is_some()));
assert!(gate(r"(?i)(?<=k)\s+y").is_some_and(|gate| gate.0.start.is_none()));
}
#[test]
fn case_insensitive_rest_bytes_cover_both_cases() {
let gate = gate(r"(?i)\s*+(select|insert)\b").expect("gate");
let mut state = SkipGateLineState::default();
assert_eq!(
gate.decide(" SELECT", 0, AnchorContext::default(), &mut state),
SkipGateDecision::Allow
);
assert_eq!(
gate.decide(" select", 0, AnchorContext::default(), &mut state),
SkipGateDecision::Allow
);
let mut state = SkipGateLineState::default();
assert_eq!(
gate.decide(" update", 0, AnchorContext::default(), &mut state),
SkipGateDecision::Skip
);
}
}