1use regex_syntax::hir::{Class, ClassUnicode, ClassUnicodeRange, Hir, HirKind};
2
3use crate::optimize::optimize;
4use crate::to_hir::{parse_fragment, HirCtx};
5use crate::{Expr, LookAround, RegexOptions, Result};
6
7#[derive(Clone, Copy, Debug, PartialEq, Eq, Default)]
10pub struct ByteSet([u64; 4]);
11
12impl ByteSet {
13 pub fn insert(&mut self, byte: u8) {
15 self.0[(byte >> 6) as usize] |= 1u64 << (byte & 63);
16 }
17
18 pub fn is_empty(&self) -> bool {
20 self.0 == [0; 4]
21 }
22
23 #[inline]
25 pub fn contains(&self, byte: u8) -> bool {
26 self.0[(byte >> 6) as usize] & (1u64 << (byte & 63)) != 0
27 }
28
29 pub fn union(&mut self, other: &Self) {
31 for (a, b) in self.0.iter_mut().zip(&other.0) {
32 *a |= *b;
33 }
34 }
35
36 pub fn intersection(&mut self, other: &Self) {
38 for (a, b) in self.0.iter_mut().zip(&other.0) {
39 *a &= *b;
40 }
41 }
42
43 #[inline]
45 pub fn is_subset(&self, other: &Self) -> bool {
46 self.0
47 .iter()
48 .zip(other.0)
49 .all(|(self_word, other_word)| self_word & !other_word == 0)
50 }
51
52 pub fn iter(&self) -> impl Iterator<Item = u8> + '_ {
54 set_bits(&self.0).map(|i| i as u8)
55 }
56
57 pub fn words(&self) -> &[u64; 4] {
59 &self.0
60 }
61
62 pub const fn from_words(words: [u64; 4]) -> Self {
64 Self(words)
65 }
66}
67
68fn bits_of(mut bits: u64) -> impl Iterator<Item = usize> {
70 core::iter::from_fn(move || {
71 (bits != 0).then(|| {
72 let bit = bits.trailing_zeros() as usize;
73 bits &= bits - 1;
75 bit
76 })
77 })
78}
79
80fn set_bits(words: &[u64]) -> impl Iterator<Item = usize> + '_ {
82 words
83 .iter()
84 .enumerate()
85 .flat_map(|(i, &word)| bits_of(word).map(move |bit| i * 64 + bit))
86}
87
88#[derive(Debug, Clone, PartialEq, Eq)]
89enum Start {
90 Definite(ByteSet),
92 MaybeEmpty(ByteSet),
94 Bail,
96}
97
98impl Start {
99 fn concat(starts: impl Iterator<Item = Start>) -> Start {
100 let mut set = ByteSet::default();
101 for start in starts {
102 match start {
103 Start::Definite(mut s) => {
104 s.union(&set);
105 return Start::Definite(s);
106 }
107 Start::MaybeEmpty(s) => set.union(&s),
108 Start::Bail => return Start::Bail,
109 }
110 }
111 Start::MaybeEmpty(set)
112 }
113
114 fn alt(starts: impl Iterator<Item = Start>) -> Start {
115 let mut set = ByteSet::default();
116 let mut definite = None;
118 for start in starts {
119 match start {
120 Start::Definite(s) => {
121 set.union(&s);
122 definite.get_or_insert(true);
123 }
124 Start::MaybeEmpty(s) => {
125 set.union(&s);
126 definite = Some(false);
127 }
128 Start::Bail => return Start::Bail,
129 }
130 }
131 match definite {
132 Some(true) => Start::Definite(set),
133 _ => Start::MaybeEmpty(set),
134 }
135 }
136
137 fn optional(self) -> Start {
138 match self {
139 Start::Definite(set) | Start::MaybeEmpty(set) => Start::MaybeEmpty(set),
140 Start::Bail => Start::Bail,
141 }
142 }
143}
144
145#[inline]
146fn first_utf8_byte(c: char) -> u8 {
147 let mut buf = [0u8; 4];
148 c.encode_utf8(&mut buf).as_bytes()[0]
149}
150
151fn char_class(c: char) -> ClassUnicode {
152 ClassUnicode::new([ClassUnicodeRange::new(c, c)])
153}
154
155fn insert_class(set: &mut ByteSet, mut class: ClassUnicode, casei: bool) -> bool {
158 if casei && class.try_case_fold_simple().is_err() {
159 return false;
160 }
161 for range in class.ranges() {
162 for c in range.start()..=range.end() {
163 set.insert(first_utf8_byte(c));
164 }
165 }
166 true
167}
168
169fn regex_syntax_start(expr: &Hir, casei: bool) -> Start {
173 match expr.kind() {
174 HirKind::Empty | HirKind::Look(_) => Start::MaybeEmpty(ByteSet::default()),
176 HirKind::Literal(l) => {
177 let Some(&first) = l.0.first() else {
178 return Start::MaybeEmpty(ByteSet::default());
179 };
180 let mut set = ByteSet::default();
181 if casei {
182 let first_char = core::str::from_utf8(&l.0)
183 .ok()
184 .and_then(|s| s.chars().next());
185 let Some(c) = first_char else {
186 return Start::Bail;
187 };
188 if !insert_class(&mut set, char_class(c), true) {
189 return Start::Bail;
190 }
191 } else {
192 set.insert(first);
193 }
194 Start::Definite(set)
195 }
196 HirKind::Class(class) => {
197 let mut set = ByteSet::default();
198 match class {
199 Class::Unicode(cls) => {
200 let folded;
203 let cls = if casei {
204 let mut cls = cls.clone();
205 if cls.try_case_fold_simple().is_err() {
206 return Start::Bail;
207 }
208 folded = cls;
209 &folded
210 } else {
211 cls
212 };
213 let mut ascii = ClassUnicode::empty();
214 for r in cls.ranges() {
215 if r.start().is_ascii() {
216 ascii.push(ClassUnicodeRange::new(r.start(), r.end().min('\x7F')));
217 }
218 if !r.end().is_ascii() {
219 let start = first_utf8_byte(r.start().max('\u{80}'));
220 for b in start..=first_utf8_byte(r.end()) {
221 set.insert(b);
222 }
223 }
224 }
225 insert_class(&mut set, ascii, false);
226 }
227 Class::Bytes(cls) => {
228 for r in cls.ranges() {
229 for i in r.start()..=r.end() {
230 set.insert(i);
231 if casei && i.is_ascii_alphabetic() {
232 set.insert(i.to_ascii_uppercase());
233 set.insert(i.to_ascii_lowercase());
234 }
235 }
236 }
237 }
238 }
239
240 if set.is_empty() {
241 Start::Bail
242 } else {
243 Start::Definite(set)
244 }
245 }
246 HirKind::Capture(c) => regex_syntax_start(&c.sub, casei),
247 HirKind::Repetition(r) => {
248 let child = regex_syntax_start(&r.sub, casei);
249 if r.min == 0 {
250 child.optional()
251 } else {
252 child
253 }
254 }
255 HirKind::Concat(c) => Start::concat(c.iter().map(|e| regex_syntax_start(e, casei))),
256 HirKind::Alternation(c) => Start::alt(c.iter().map(|e| regex_syntax_start(e, casei))),
257 }
258}
259
260fn fancy_start(expr: &Expr, ctx: &mut HirCtx) -> Start {
261 match expr {
262 Expr::Empty | Expr::Assertion(_) | Expr::KeepOut | Expr::ContinueFromPreviousMatchEnd => {
263 Start::MaybeEmpty(ByteSet::default())
264 }
265 Expr::Literal { val, casei } => {
266 let Some(first) = val.chars().next() else {
267 return Start::MaybeEmpty(ByteSet::default());
268 };
269 if !ctx.utf8() && !first.is_ascii() {
272 return Start::Bail;
273 }
274 let mut set = ByteSet::default();
275 if !insert_class(&mut set, char_class(first), *casei) {
276 return Start::Bail;
277 }
278 Start::Definite(set)
279 }
280 Expr::Delegate { inner, casei } => match parse_fragment(inner, ctx) {
281 Some(hir) => regex_syntax_start(&hir, *casei),
282 None => Start::Bail,
283 },
284
285 Expr::Concat(exprs) => Start::concat(exprs.iter().map(|e| fancy_start(e, ctx))),
286 Expr::Alt(exprs) => Start::alt(exprs.iter().map(|e| fancy_start(e, ctx))),
287 Expr::Group(child) => fancy_start(child, ctx),
288 Expr::AtomicGroup(child) => fancy_start(child, ctx),
289 Expr::Repeat { child, lo, .. } => {
290 let child = fancy_start(child, ctx);
291 if *lo == 0 {
292 child.optional()
293 } else {
294 child
295 }
296 }
297 Expr::LookAround(expr, lookaround) => match lookaround {
298 LookAround::LookAhead => match fancy_start(expr, ctx) {
299 Start::Definite(set) => Start::Definite(set),
300 _ => Start::MaybeEmpty(ByteSet::default()),
301 },
302 _ => Start::MaybeEmpty(ByteSet::default()),
303 },
304 _ => Start::Bail,
305 }
306}
307
308pub(crate) fn byte_set_from_expr(expr: &Expr, options: &RegexOptions) -> Option<ByteSet> {
309 let mut hir_ctx = HirCtx::from(options);
310 match fancy_start(expr, &mut hir_ctx) {
311 Start::Definite(set) if !set.is_empty() => Some(set),
312 _ => None,
313 }
314}
315
316pub(crate) fn start_bytes(pattern: &str, options: &RegexOptions) -> Result<Option<ByteSet>> {
318 let mut tree = Expr::parse_tree_with_flags(pattern, options.compute_flags())?;
319 optimize(&mut tree);
322 Ok(byte_set_from_expr(&tree.expr, options))
323}
324
325fn required_byte_set_from_expr(expr: &Expr) -> ByteSet {
326 let mut byte_set = ByteSet::default();
327
328 match expr {
329 Expr::Literal { val, casei } => {
330 for byte in val.bytes() {
331 if !casei || (byte.is_ascii() && !byte.is_ascii_alphabetic()) {
337 byte_set.insert(byte);
338 }
339 }
340 }
341 Expr::LiteralBytes { .. } => {
342 }
346 Expr::Concat(children) => {
347 for child in children {
348 byte_set.union(&required_byte_set_from_expr(child));
349 }
350 }
351 Expr::Alt(children) => {
352 if let Some((first, rest)) = children.split_first() {
353 byte_set = required_byte_set_from_expr(first);
354 for child in rest {
355 let other = required_byte_set_from_expr(child);
356 byte_set.intersection(&other);
357 }
358 }
359 }
360 Expr::Conditional {
361 condition,
362 true_branch,
363 false_branch,
364 } => {
365 byte_set = required_byte_set_from_expr(true_branch);
368 byte_set.union(&required_byte_set_from_expr(condition));
369 let other = required_byte_set_from_expr(false_branch);
370 byte_set.intersection(&other);
371 }
372 Expr::Group(child) => return required_byte_set_from_expr(child),
373 Expr::AtomicGroup(child)
374 | Expr::LookAround(child, LookAround::LookAhead | LookAround::LookBehind)
375 | Expr::Repeat { child, lo: 1.., .. } => return required_byte_set_from_expr(child),
376 Expr::Empty
377 | Expr::Assertion(_)
378 | Expr::DefineGroup { .. }
379 | Expr::KeepOut
380 | Expr::ContinueFromPreviousMatchEnd
381 | Expr::BackrefExistsCondition { .. }
382 | Expr::BacktrackingControlVerb(_)
383 | Expr::Any { .. }
384 | Expr::GeneralNewline { .. }
385 | Expr::Delegate { .. }
386 | Expr::Repeat { .. }
387 | Expr::LookAround(..)
388 | Expr::Absent(_)
389 | Expr::Backref { .. }
390 | Expr::BackrefWithRelativeRecursionLevel { .. }
391 | Expr::SubroutineCall(_)
392 | Expr::AstNode(..) => {}
393 }
394
395 byte_set
396}
397
398pub(crate) fn required_bytes(pattern: &str, options: &RegexOptions) -> Result<ByteSet> {
400 let mut tree = Expr::parse_tree_with_flags(pattern, options.compute_flags())?;
401 optimize(&mut tree);
402 Ok(required_byte_set_from_expr(&tree.expr))
403}
404
405#[cfg(test)]
406mod tests {
407 use super::*;
408 use crate::{RegexInput, RegexOptionsBuilder};
409 use alloc::borrow::ToOwned;
410 use alloc::vec;
411 use alloc::vec::Vec;
412
413 fn get_byte_set_from_pattern(pattern: &str, options: &RegexOptions) -> Option<ByteSet> {
414 start_bytes(pattern, options).ok().flatten()
415 }
416
417 fn get_required_byte_set_from_pattern(pattern: &str) -> ByteSet {
418 let mut byte_set = ByteSet::default();
419 for b in pattern.bytes() {
420 byte_set.insert(b);
421 }
422 byte_set
423 }
424
425 fn first_bytes_of(pattern: &str) -> Option<Vec<u8>> {
426 let set = get_byte_set_from_pattern(pattern, &RegexOptions::default())?;
427 Some(set.iter().collect())
428 }
429
430 #[test]
431 fn can_extract_ascii_byteset_from_pattern() {
432 let inputs = vec![
433 (r"fn", Some(vec![b'f'])),
434 (r"::", Some(vec![b':'])),
435 (r"=>", Some(vec![b'='])),
436 (r"\{", Some(vec![b'{'])),
437 (r"fn|let|impl", Some(vec![b'f', b'i', b'l'])),
438 (r"(fn|let|impl)", Some(vec![b'f', b'i', b'l'])),
439 (r"(?:->|<-)", Some(vec![b'-', b'<'])),
440 (r"\bfn\b", Some(vec![b'f'])),
441 (r"^let", Some(vec![b'l'])),
442 (r"\Gx", Some(vec![b'x'])),
443 (r"(?<!\.)let", Some(vec![b'l'])),
444 (r"[abc]x", Some(vec![b'a', b'b', b'c'])),
445 (r"[a-d]", Some(vec![b'a', b'b', b'c', b'd'])),
446 (r"(?i)fn", Some(vec![b'F', b'f'])),
447 (r"(?i)[a-b]", Some(vec![b'A', b'B', b'a', b'b'])),
448 (r"&(?![&=])", Some(vec![b'&'])),
449 (r"(?x) fn # comment", Some(vec![b'f'])),
450 (r"(?=\[)", Some(vec![b'['])),
451 (r"(?=.*)x", Some(vec![b'x'])),
452 (r"(?=a?)x", Some(vec![b'x'])),
453 (r"(?!x)a", Some(vec![b'a'])),
454 (r#"(?=["'`])""#, Some(vec![b'"', b'\'', b'`'])),
455 (r#"(?=ab|xy)"#, Some(vec![b'a', b'x'])),
456 (r#"r?""#, Some(vec![b'"', b'r'])),
457 (r"a+", Some(vec![b'a'])),
458 (r"(?>fn)", Some(vec![b'f'])),
459 (r"\Kfn", Some(vec![b'f'])),
460 (r"über", Some(vec![0xC3])),
461 (r"[α-ω]x", Some(vec![0xCE, 0xCF])),
463 (r"a?", None),
464 (r"(?!x)", None),
465 (r".", None),
466 (r".*x", None),
467 (r"\R", None),
468 (r"(", None),
469 (r"(?i)sort", Some(vec![b'S', b's', 0xC5])),
472 (r"(?i)kind", Some(vec![b'K', b'k', 0xE2])),
474 (r"(?i)ſ", Some(vec![b'S', b's', 0xC5])),
476 (r"(?i)é", Some(vec![0xC3])),
478 (r"(?i)[k]", Some(vec![b'K', b'k', 0xE2])),
480 (r"(?i)[s-t]x", Some(vec![b'S', b'T', b's', b't', 0xC5])),
481 (r"(?i)[à-ÿ]", Some(vec![0xC3, 0xC5, 0xE2])),
484 (r"(?i)[α-ω]", Some(vec![0xC2, 0xCD, 0xCE, 0xCF, 0xE1, 0xE2])),
485 (r"(?i)[0-9_]", Some(b"0123456789_".to_vec())),
487 ];
488
489 for (input, expected) in inputs {
490 assert_eq!(first_bytes_of(input), expected, "pattern {input:?}");
491 }
492 }
493
494 #[test]
495 fn can_extract_byteset_from_unicode() {
496 let upper = first_bytes_of(r"\p{upper}!").unwrap();
497 assert!(upper.contains(&b'A') && upper.contains(&b'Z'));
498 assert!(!upper.contains(&b'a') && !upper.contains(&b'!'));
499 assert!(upper.contains(&0xC3));
501
502 let not_upper = first_bytes_of(r"\P{upper}!").unwrap();
503 assert!(not_upper.contains(&b'a') && not_upper.contains(&b'z'));
504 assert!(!not_upper.contains(&b'A') && not_upper.contains(&b'!'));
506 assert!(first_bytes_of(r"\P{typo}!").is_none());
508 }
509
510 #[test]
511 fn can_extract_byteset_from_negation() {
512 let b1 = first_bytes_of(r"[^a-c]x").unwrap();
513 assert!(!b1.contains(&b'a') && !b1.contains(&b'c'));
514 assert!(b1.contains(&b'd') && b1.contains(&b' ') && b1.contains(&0xC3));
515
516 let b2 = first_bytes_of(r"(?i)[^a]").unwrap();
517 assert!(b2.contains(&b'b') && b2.contains(&b'B') && b2.contains(&0xC3));
518
519 let b3 = first_bytes_of(r"[^[:alpha:]]").unwrap();
520 assert!(!b3.contains(&b'a') && !b3.contains(&b'Z'));
521 assert!(b3.contains(&b'0') && b3.contains(&b' ') && b3.contains(&0xC3));
522 }
523
524 #[test]
525 fn non_ascii_only_folds_to_k_and_s() {
526 let mut class = ClassUnicode::new([ClassUnicodeRange::new('\u{80}', '\u{10FFFF}')]);
527 class.try_case_fold_simple().unwrap();
528 let ascii: Vec<char> = class
529 .ranges()
530 .iter()
531 .flat_map(|r| r.start()..=r.end())
532 .filter(char::is_ascii)
533 .collect();
534 assert_eq!(ascii, vec!['K', 'S', 'k', 's']);
535 }
536
537 #[test]
538 fn can_extract_required_bytes() {
539 let inputs = vec![
540 ("a?b", "b"),
541 ("ab|ac", "a"),
542 ("a|b", ""),
543 ("(?=a)b", "ab"),
544 ("(?<=a)b", "ab"),
545 ("(?!a)b", "b"),
546 ("(?i)k!", "!"),
547 ("(?(DEFINE)(a))b", "b"),
548 ("(a){0}b", "b"),
549 ("é!", "é!"),
550 ("(?i)é!", "!"),
551 ("(?i:É)!", "!"),
552 ("(x)?(?(1)b!|c!)", "!"),
553 ("(x)(?(1)ab|ac)", "ax"),
554 ("(x)(?(1)ab)", "x"),
555 ("(?(a)b|ac)", "a"),
556 ("(?(a)b|zc)", ""),
557 ("(?(a)b|b)", "b"),
558 ];
559
560 for (pattern, bytes) in inputs {
561 assert_eq!(
562 required_bytes(pattern, &RegexOptions::default()).unwrap(),
563 get_required_byte_set_from_pattern(bytes),
564 "expected byte set for pattern {} doesn't match",
565 pattern
566 );
567 }
568 }
569
570 #[test]
571 fn required_bytes_match_match_result() {
572 let inputs = vec![
573 "a?b",
574 "ab|ac",
575 "a|b",
576 "(?=a)a",
577 "(?<=a)b",
578 "(?!a)b",
579 "(?i)k!",
580 "(?i)s",
581 "(?i:ABC)!",
582 r"a\Kb",
583 r"(a)\1",
584 "(a){0}b",
585 "(?x)a # ignored\n b",
586 "a*",
587 "^a$",
588 r"\Ga",
589 r"\Aa",
590 r"(?<x>a)(?(x)b|c)",
591 r"(x)?(?(1)b!|c!)",
592 r"(x)(?(1)ab|ac)",
593 r"(x)(?(1)ab)",
594 "(?i)é!",
595 "(?i:É)!",
596 "日!",
597 r"(?<x>a)\g<x>",
598 "é!",
599 ];
600
601 for pattern in inputs {
602 let re = RegexOptionsBuilder::default()
603 .build(pattern.to_owned())
604 .unwrap();
605
606 for text in [
609 "", "a", "b", "ab", "ac", "aa", "abc!", "ABC!", "K!", "ſ", "é!", "a\nb", "É!",
610 "日!", "x", "xb!", "xc!", "xab", "xac", "b!", "c!",
611 ] {
612 for pos in (0..=text.len()).filter(|&p| text.is_char_boundary(p)) {
613 let input = RegexInput::new(text).from_pos(pos).anchored(true);
614 if re.find_input(input).unwrap().is_some() {
615 assert!(required_bytes(pattern, &RegexOptions::default())
616 .unwrap()
617 .iter()
618 .all(|byte| get_required_byte_set_from_pattern(text).contains(byte)));
619 }
620 }
621 }
622 }
623 }
624
625 #[test]
626 fn can_do_set_operations() {
627 fn set(bytes: &[u8]) -> ByteSet {
628 let mut set = ByteSet::default();
629 for &b in bytes {
630 set.insert(b);
631 }
632 set
633 }
634
635 let empty = ByteSet::default();
636 let small = set(&[0, 70, 140]);
637 let big = set(&[0, 70, 140, 200, 255]);
638 let other = set(&[70, 140, 210]);
639
640 assert!(empty.is_subset(&small));
641 assert!(small.is_subset(&small));
642 assert!(small.is_subset(&big));
643 assert!(!big.is_subset(&small));
644 assert!(!other.is_subset(&big));
645
646 let mut intersection = big;
647 intersection.intersection(&other);
648 assert_eq!(intersection, set(&[70, 140]));
649
650 let mut union = small;
651 union.union(&other);
652 assert_eq!(union, set(&[0, 70, 140, 210]));
653 }
654}