rsigma_convert/reverse/
lucene.rs

1//! Elastic Lucene reverse frontend.
2//!
3//! Parses the Lucene / Elasticsearch `query_string` subset used by detection
4//! authors into the HIR: `field:value` (with `*`/`?` wildcards), quoted
5//! phrases, `field:/regex/`, `field:[a TO b]` / `{a TO b}` ranges,
6//! `field:>=N` comparisons, `field:(a OR b)` value groups, `_exists_:field`,
7//! bare keyword terms, and the `AND`/`OR`/`NOT` (plus `&&`/`||`/`!` and
8//! `+`/`-`) boolean operators with parentheses.
9//!
10//! Adjacent terms with no explicit operator are ANDed
11//! ([`QueryDialect::implicit_and`]), matching a `default_operator: AND` posture,
12//! which is the common intent for detection queries.
13//!
14//! Constructs with no Sigma equivalent are rejected with a structured
15//! [`ConvertError::UnsupportedConstruct`]: boosting (`^`), fuzzy and proximity
16//! (`~`), and non-numeric ranges.
17
18use rsigma_ir::{IrMatcher, IrNumber, IrPattern, IrPatternPart};
19
20use crate::error::{ConvertError, Result};
21
22use super::{
23    Frontend, QueryDialect, QueryExpr, QueryLeaf, ReverseCtx, infer_str_matcher, parse_pattern,
24};
25
26/// The Lucene boolean dialect.
27pub const LUCENE_DIALECT: QueryDialect = QueryDialect {
28    name: "lucene",
29    and_tokens: &["AND", "&&"],
30    or_tokens: &["OR", "||"],
31    not_tokens: &["NOT", "!"],
32    implicit_and: true,
33};
34
35/// Reverse frontend for Elastic Lucene query strings.
36#[derive(Debug, Default, Clone, Copy)]
37pub struct LuceneFrontend;
38
39impl Frontend for LuceneFrontend {
40    fn name(&self) -> &str {
41        LUCENE_DIALECT.name
42    }
43
44    fn dialect(&self) -> &QueryDialect {
45        &LUCENE_DIALECT
46    }
47
48    fn parse_atom(&self, atom: &str, ctx: &ReverseCtx) -> Result<QueryExpr<QueryLeaf>> {
49        if let Some((field, value)) = split_field(atom) {
50            if field == "_exists_" {
51                return Ok(leaf(field_leaf(unquote(value), IrMatcher::Exists(true))));
52            }
53            if is_field_name(field) {
54                return parse_field(field, value.trim(), ctx);
55            }
56        }
57        Ok(leaf(QueryLeaf::Keyword(keyword_matcher(atom)?)))
58    }
59}
60
61// =============================================================================
62// Field predicate parsing
63// =============================================================================
64
65fn parse_field(field: &str, value: &str, ctx: &ReverseCtx) -> Result<QueryExpr<QueryLeaf>> {
66    if let Some(inner) = strip_pair(value, '(', ')') {
67        return parse_value_group(field, inner, ctx);
68    }
69    if value.starts_with('[') || value.starts_with('{') {
70        return parse_range(field, value);
71    }
72    if let Some(pattern) = strip_pair(value, '/', '/') {
73        return Ok(leaf(field_leaf(
74            field.to_string(),
75            IrMatcher::Regex {
76                pattern: regex_unescape(pattern),
77                case_insensitive: false,
78                multiline: false,
79                dotall: false,
80                cased: false,
81            },
82        )));
83    }
84    if let Some(matcher) = parse_comparison(value)? {
85        return Ok(leaf(field_leaf(field.to_string(), matcher)));
86    }
87    Ok(leaf(field_leaf(field.to_string(), scalar_matcher(value)?)))
88}
89
90/// Parse a `field:(a OR b ...)` value group by reusing the shared boolean parser
91/// over the bare values, all bound to `field`.
92fn parse_value_group(field: &str, inner: &str, _ctx: &ReverseCtx) -> Result<QueryExpr<QueryLeaf>> {
93    let tokens = super::tokenize(&LUCENE_DIALECT, inner)?;
94    let tree = super::parse_boolean(&LUCENE_DIALECT, tokens)?;
95    tree.expand(&mut |atom: String| {
96        Ok(QueryExpr::Leaf(field_leaf(
97            field.to_string(),
98            scalar_matcher(&atom)?,
99        )))
100    })
101}
102
103/// Parse a `[a TO b]` (inclusive) or `{a TO b}` (exclusive) numeric range.
104fn parse_range(field: &str, value: &str) -> Result<QueryExpr<QueryLeaf>> {
105    let open = value.chars().next().unwrap_or('[');
106    let close = value.chars().last().unwrap_or(']');
107    let inner = &value[1..value.len().saturating_sub(1)];
108
109    let (low, high) = split_range(inner).ok_or_else(|| {
110        ConvertError::QueryParse(format!("range must be '[low TO high]', got '{value}'"))
111    })?;
112
113    let inclusive_low = open == '[';
114    let inclusive_high = close == ']';
115
116    let mut bounds = Vec::new();
117    if low != "*" {
118        let n = range_number(low)?;
119        bounds.push(if inclusive_low {
120            IrMatcher::NumericGte(n)
121        } else {
122            IrMatcher::NumericGt(n)
123        });
124    }
125    if high != "*" {
126        let n = range_number(high)?;
127        bounds.push(if inclusive_high {
128            IrMatcher::NumericLte(n)
129        } else {
130            IrMatcher::NumericLt(n)
131        });
132    }
133
134    match bounds.len() {
135        0 => Ok(leaf(field_leaf(field.to_string(), IrMatcher::Exists(true)))),
136        1 => Ok(leaf(field_leaf(field.to_string(), bounds.pop().unwrap()))),
137        _ => Ok(QueryExpr::And(
138            bounds
139                .into_iter()
140                .map(|m| leaf(field_leaf(field.to_string(), m)))
141                .collect(),
142        )),
143    }
144}
145
146fn split_range(inner: &str) -> Option<(&str, &str)> {
147    for sep in [" TO ", " to "] {
148        if let Some(idx) = inner.find(sep) {
149            let low = inner[..idx].trim();
150            let high = inner[idx + sep.len()..].trim();
151            return Some((low, high));
152        }
153    }
154    None
155}
156
157fn range_number(s: &str) -> Result<IrNumber> {
158    s.parse::<f64>().map(IrNumber::Literal).map_err(|_| {
159        ConvertError::UnsupportedConstruct(format!(
160            "non-numeric range bound '{s}' has no Sigma equivalent"
161        ))
162    })
163}
164
165fn parse_comparison(value: &str) -> Result<Option<IrMatcher>> {
166    let (ctor, rest): (fn(IrNumber) -> IrMatcher, &str) = if let Some(r) = value.strip_prefix(">=")
167    {
168        (IrMatcher::NumericGte, r)
169    } else if let Some(r) = value.strip_prefix("<=") {
170        (IrMatcher::NumericLte, r)
171    } else if let Some(r) = value.strip_prefix('>') {
172        (IrMatcher::NumericGt, r)
173    } else if let Some(r) = value.strip_prefix('<') {
174        (IrMatcher::NumericLt, r)
175    } else {
176        return Ok(None);
177    };
178    let n = rest.trim().parse::<f64>().map_err(|_| {
179        ConvertError::UnsupportedConstruct(format!("non-numeric comparison bound '{rest}'"))
180    })?;
181    Ok(Some(ctor(IrNumber::Literal(n))))
182}
183
184/// Interpret a single scalar value (not a range, group, regex, or comparison).
185fn scalar_matcher(value: &str) -> Result<IrMatcher> {
186    if let Some(inner) = quoted_inner(value) {
187        return Ok(IrMatcher::Str {
188            op: rsigma_ir::IrStrOp::Exact,
189            pattern: plain_pattern(&inner),
190            case_insensitive: true,
191        });
192    }
193
194    reject_boost_fuzzy(value)?;
195
196    match value {
197        "true" => return Ok(IrMatcher::BoolEq(true)),
198        "false" => return Ok(IrMatcher::BoolEq(false)),
199        _ => {}
200    }
201
202    let has_wildcard = has_unescaped_wildcard(value);
203    if !has_wildcard {
204        if let Ok(n) = value.parse::<i64>() {
205            return Ok(IrMatcher::NumericEq(IrNumber::Literal(n as f64)));
206        }
207        if let Ok(f) = value.parse::<f64>() {
208            return Ok(IrMatcher::NumericEq(IrNumber::Literal(f)));
209        }
210    }
211
212    Ok(infer_str_matcher(&lucene_value_to_sigma(value), true))
213}
214
215fn keyword_matcher(term: &str) -> Result<IrMatcher> {
216    if let Some(inner) = quoted_inner(term) {
217        return Ok(IrMatcher::Str {
218            op: rsigma_ir::IrStrOp::Contains,
219            pattern: plain_pattern(&inner),
220            case_insensitive: true,
221        });
222    }
223    reject_boost_fuzzy(term)?;
224    Ok(IrMatcher::Str {
225        op: rsigma_ir::IrStrOp::Contains,
226        pattern: parse_pattern(&lucene_value_to_sigma(term)),
227        case_insensitive: true,
228    })
229}
230
231// =============================================================================
232// Helpers
233// =============================================================================
234
235fn leaf(l: QueryLeaf) -> QueryExpr<QueryLeaf> {
236    QueryExpr::Leaf(l)
237}
238
239fn field_leaf(field: String, matcher: IrMatcher) -> QueryLeaf {
240    QueryLeaf::Field { field, matcher }
241}
242
243fn plain_pattern(s: &str) -> IrPattern {
244    IrPattern {
245        parts: if s.is_empty() {
246            Vec::new()
247        } else {
248            vec![IrPatternPart::Literal(s.to_string())]
249        },
250    }
251}
252
253/// Split an atom at its first top-level `:` (not inside quotes or brackets).
254fn split_field(atom: &str) -> Option<(&str, &str)> {
255    let chars: Vec<(usize, char)> = atom.char_indices().collect();
256    let mut depth = 0usize;
257    let mut quote: Option<char> = None;
258    for (byte_idx, c) in &chars {
259        match quote {
260            Some(q) => {
261                if *c == q {
262                    quote = None;
263                }
264            }
265            None => match c {
266                '"' | '\'' => quote = Some(*c),
267                '(' | '[' | '{' => depth += 1,
268                ')' | ']' | '}' => depth = depth.saturating_sub(1),
269                ':' if depth == 0 => {
270                    return Some((&atom[..*byte_idx], &atom[byte_idx + 1..]));
271                }
272                _ => {}
273            },
274        }
275    }
276    None
277}
278
279fn is_field_name(name: &str) -> bool {
280    let mut chars = name.chars();
281    match chars.next() {
282        Some(c) if c.is_ascii_alphabetic() || c == '_' || c == '@' => {}
283        _ => return false,
284    }
285    name.chars()
286        .all(|c| c.is_ascii_alphanumeric() || matches!(c, '_' | '.' | '-' | '@'))
287}
288
289fn strip_pair(value: &str, open: char, close: char) -> Option<&str> {
290    let mut chars = value.chars();
291    if chars.next()? != open {
292        return None;
293    }
294    if value.chars().count() < 2 || !value.ends_with(close) {
295        return None;
296    }
297    let start = open.len_utf8();
298    let end = value.len() - close.len_utf8();
299    Some(&value[start..end])
300}
301
302fn quoted_inner(value: &str) -> Option<String> {
303    for q in ['"', '\''] {
304        if value.len() >= 2 && value.starts_with(q) && value.ends_with(q) {
305            let inner = &value[1..value.len() - 1];
306            // A quoted phrase is a literal: unescape Lucene backslash escapes
307            // (`\\` -> `\`, `\"` -> `"`, ...) so the phrase is the intended text.
308            let mut out = String::with_capacity(inner.len());
309            let mut chars = inner.chars();
310            while let Some(c) = chars.next() {
311                if c == '\\' {
312                    if let Some(next) = chars.next() {
313                        out.push(next);
314                    }
315                } else {
316                    out.push(c);
317                }
318            }
319            return Some(out);
320        }
321    }
322    None
323}
324
325fn unquote(value: &str) -> String {
326    quoted_inner(value).unwrap_or_else(|| value.to_string())
327}
328
329fn has_unescaped_wildcard(value: &str) -> bool {
330    let mut chars = value.chars().peekable();
331    while let Some(c) = chars.next() {
332        match c {
333            '\\' => {
334                chars.next();
335            }
336            '*' | '?' => return true,
337            _ => {}
338        }
339    }
340    false
341}
342
343fn reject_boost_fuzzy(value: &str) -> Result<()> {
344    let mut chars = value.chars();
345    while let Some(c) = chars.next() {
346        match c {
347            '\\' => {
348                chars.next();
349            }
350            '^' => {
351                return Err(ConvertError::UnsupportedConstruct(
352                    "term boosting (^) has no Sigma equivalent".into(),
353                ));
354            }
355            '~' => {
356                return Err(ConvertError::UnsupportedConstruct(
357                    "fuzzy/proximity (~) has no Sigma equivalent".into(),
358                ));
359            }
360            _ => {}
361        }
362    }
363    Ok(())
364}
365
366/// Convert a Lucene value into a Sigma value string: keep `\*`/`\?`/`\\`
367/// escapes (so wildcards stay literal), and drop backslashes escaping other
368/// Lucene specials (so `\:` becomes a literal `:`).
369fn lucene_value_to_sigma(value: &str) -> String {
370    let mut out = String::with_capacity(value.len());
371    let chars: Vec<char> = value.chars().collect();
372    let mut i = 0;
373    while i < chars.len() {
374        if chars[i] == '\\' && i + 1 < chars.len() {
375            let next = chars[i + 1];
376            match next {
377                '*' | '?' | '\\' => {
378                    out.push('\\');
379                    out.push(next);
380                }
381                _ => out.push(next),
382            }
383            i += 2;
384        } else {
385            out.push(chars[i]);
386            i += 1;
387        }
388    }
389    out
390}
391
392fn regex_unescape(pattern: &str) -> String {
393    pattern.replace("\\/", "/")
394}
395
396#[cfg(test)]
397mod tests {
398    use super::*;
399    use crate::reverse::ReverseCtx;
400
401    fn rule_yaml(query: &str) -> String {
402        let frontend = LuceneFrontend;
403        let ctx = ReverseCtx {
404            title: Some("Test".into()),
405            product: Some("windows".into()),
406            ..Default::default()
407        };
408        let ir = frontend.parse_query(query, &ctx).expect("parses");
409        let rule = rsigma_ir::raise_rule(&ir, &rsigma_ir::RaiseOptions::default()).expect("raises");
410        rsigma_parser::emit_rule_yaml(&rule)
411    }
412
413    #[test]
414    fn simple_field_equality() {
415        let out = rule_yaml("EventID:4688");
416        assert!(out.contains("EventID: 4688"), "{out}");
417    }
418
419    #[test]
420    fn wildcards_become_modifiers() {
421        let out = rule_yaml("Image:*\\\\cmd.exe");
422        assert!(out.contains("Image|endswith:"), "{out}");
423    }
424
425    #[test]
426    fn regex_maps_to_re_modifier() {
427        let out = rule_yaml("CommandLine:/a.*b/");
428        assert!(out.contains("CommandLine|re: 'a.*b'"), "{out}");
429    }
430
431    #[test]
432    fn inclusive_range_becomes_gte_lte() {
433        let out = rule_yaml("Port:[1024 TO 65535]");
434        assert!(out.contains("Port|gte: 1024"), "{out}");
435        assert!(out.contains("Port|lte: 65535"), "{out}");
436    }
437
438    #[test]
439    fn comparison_shorthand() {
440        let out = rule_yaml("Port:>=1024");
441        assert!(out.contains("Port|gte: 1024"), "{out}");
442    }
443
444    #[test]
445    fn value_group_becomes_value_list() {
446        let out = rule_yaml("Image:(\"a.exe\" OR \"b.exe\")");
447        assert!(out.contains("Image:"), "{out}");
448        assert!(out.contains("- a.exe"), "{out}");
449        assert!(out.contains("- b.exe"), "{out}");
450    }
451
452    #[test]
453    fn exists_maps_to_exists_modifier() {
454        let out = rule_yaml("_exists_:CommandLine");
455        assert!(out.contains("CommandLine|exists: true"), "{out}");
456    }
457
458    #[test]
459    fn keyword_term_becomes_keywords_selection() {
460        let out = rule_yaml("mimikatz");
461        assert!(out.contains("keywords:"), "{out}");
462        assert!(out.contains("- mimikatz"), "{out}");
463    }
464
465    #[test]
466    fn boosting_is_rejected() {
467        let frontend = LuceneFrontend;
468        let err = frontend
469            .parse_query("field:value^2", &ReverseCtx::default())
470            .unwrap_err();
471        assert!(matches!(err, ConvertError::UnsupportedConstruct(_)));
472    }
473
474    #[test]
475    fn fuzzy_is_rejected() {
476        let frontend = LuceneFrontend;
477        let err = frontend
478            .parse_query("roam~", &ReverseCtx::default())
479            .unwrap_err();
480        assert!(matches!(err, ConvertError::UnsupportedConstruct(_)));
481    }
482}
rsigma_convert/reverse/lucene.rs

rsigma_convert/reverse/
lucene.rs
