summa_core/query/
regex.rs

1//! Bounded whole-term regular-expression filters.
2use super::term_pattern::{TermPatternQuery, check_length};
3#[cfg(feature = "sync")]
4use super::traits::Scorer;
5use super::traits::{CountFuture, Query, ScorerFuture};
6use crate::dsl::Field;
7use crate::segment::SegmentReader;
8use crate::{Error, Result};
9
10/// Constant-score union of indexed terms matching a whole regular expression.
11///
12/// Supports literals, classes/ranges, grouping, alternation, `.`, `?`, `*`, `+`
13/// and bounded repetition. Matching is case-sensitive and Unicode-aware; patterns
14/// are not analyzed. Extended Lucene operators and regex-engine extensions are
15/// rejected. Existing wildcard dictionary/posting limits apply.
16#[derive(Debug, Clone)]
17pub struct RegexQuery(TermPatternQuery);
18
19impl RegexQuery {
20    pub fn new(field: Field, pattern: impl AsRef<str>) -> Result<Self> {
21        let source = pattern.as_ref();
22        check_length(source, "regex")?;
23        validate_syntax(source)?;
24        Ok(Self(TermPatternQuery::compile(
25            field,
26            source,
27            source,
28            literal_prefixes(source)?,
29            "regex",
30        )?))
31    }
32}
33
34/// Every matching term must start with one extracted literal. Limits make an
35/// infinite set fall back to the full field; they never truncate its language.
36fn literal_prefixes(source: &str) -> Result<Vec<Vec<u8>>> {
37    let hir = regex_syntax::ParserBuilder::new()
38        .dot_matches_new_line(true)
39        .build()
40        .parse(source)
41        .map_err(|error| Error::Query(format!("invalid regex pattern: {error}")))?;
42    let sequence = regex_syntax::hir::literal::Extractor::new()
43        .limit_total(64)
44        .limit_literal_len(64)
45        .limit_repeat(8)
46        .extract(&hir);
47    let Some(literals) = sequence.literals() else {
48        return Ok(vec![Vec::new()]);
49    };
50    let mut prefixes: Vec<_> = literals
51        .iter()
52        .map(|literal| literal.as_bytes().to_vec())
53        .collect();
54    prefixes.sort_unstable();
55    let mut disjoint: Vec<Vec<u8>> = Vec::with_capacity(prefixes.len());
56    for prefix in prefixes {
57        if disjoint
58            .last()
59            .is_none_or(|previous| !prefix.starts_with(previous))
60        {
61            disjoint.push(prefix);
62        }
63    }
64    Ok(disjoint)
65}
66
67fn validate_syntax(source: &str) -> Result<()> {
68    let unsupported = || {
69        Error::Query(
70            "unsupported regex syntax; use literals, classes, groups, alternation and repetition"
71                .into(),
72        )
73    };
74    let mut chars = source.chars().peekable();
75    let mut class = false;
76    while let Some(ch) = chars.next() {
77        match ch {
78            '\\' => {
79                let escaped = chars
80                    .next()
81                    .ok_or_else(|| Error::Query("regex pattern ends with an escape".into()))?;
82                if escaped.is_alphanumeric() {
83                    return Err(unsupported());
84                }
85            }
86            '[' if class => return Err(unsupported()),
87            '[' => class = true,
88            ']' => class = false,
89            '&' | '~' | '#' | '@' | '<' | '>' | '"' if !class => return Err(unsupported()),
90            '^' | '$' if !class => return Err(unsupported()),
91            '(' if !class && chars.peek() == Some(&'?') => return Err(unsupported()),
92            '&' | '~' | '|' | '-' if class && chars.peek() == Some(&ch) => {
93                return Err(unsupported());
94            }
95            _ => {}
96        }
97    }
98    Ok(())
99}
100
101impl std::fmt::Display for RegexQuery {
102    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
103        self.0.fmt(f)
104    }
105}
106impl Query for RegexQuery {
107    fn scorer<'a>(&self, reader: &'a SegmentReader, limit: usize) -> ScorerFuture<'a> {
108        self.0.scorer(reader, limit)
109    }
110    #[cfg(feature = "sync")]
111    fn scorer_sync<'a>(
112        &self,
113        reader: &'a SegmentReader,
114        limit: usize,
115    ) -> Result<Box<dyn Scorer + 'a>> {
116        self.0.scorer_sync(reader, limit)
117    }
118    fn count_estimate<'a>(&self, reader: &'a SegmentReader) -> CountFuture<'a> {
119        self.0.count_estimate(reader)
120    }
121    fn is_filter(&self) -> bool {
122        true
123    }
124}
125
126#[cfg(test)]
127mod tests {
128    #[test]
129    fn extracted_ranges_keep_alternation_optional_prefixes_and_unicode() {
130        // The prior planner always scanned the entire field, even for these
131        // selective, syntactically proven ranges.
132        let ranges = |source| super::literal_prefixes(source).unwrap();
133        assert_eq!(ranges("colou?r"), [b"color".to_vec(), b"colour".to_vec()]);
134        assert_eq!(
135            ranges("(www|http|https)"),
136            [b"http".to_vec(), b"www".to_vec()]
137        );
138        assert_eq!(ranges("(alpha|alphabet).*"), [b"alpha".to_vec()]);
139        assert_eq!(ranges("a?b"), [b"ab".to_vec(), b"b".to_vec()]);
140        assert_eq!(ranges(".*tion"), [Vec::<u8>::new()]);
141        assert_eq!(
142            ranges("(é|🦀)x"),
143            ["éx".as_bytes().to_vec(), "🦀x".as_bytes().to_vec()]
144        );
145        let regex = regex::Regex::new("^(?:[ab]{50})$").unwrap();
146        let prefixes = ranges("[ab]{50}");
147        assert!(prefixes.len() <= 64);
148        for term in ["a".repeat(50), "b".repeat(50), "ab".repeat(25)] {
149            assert!(regex.is_match(&term));
150            assert!(
151                prefixes
152                    .iter()
153                    .any(|prefix| term.as_bytes().starts_with(prefix))
154            );
155        }
156    }
157}
summa_core/query/regex.rs

summa_core/query/
regex.rs
