//! TQL AST to OpenSearch Query DSL translator.
//!
//! This module translates TQL abstract syntax trees into OpenSearch Query DSL.
use super::error::{OpenSearchError, Result};
use super::field_mappings::{FieldMappings, FieldType};
use crate::parser::{
AstNode, CollectionOpNode, ComparisonNode, NslookupExprNode, Value as AstValue,
};
use crate::regex_compat::to_lucene_regex;
use serde_json::{json, Value as JsonValue};
/// Query builder for translating TQL to OpenSearch DSL
pub struct QueryBuilder {
field_mappings: Option<FieldMappings>,
}
/// The Painless scripts for the `all` / `not_all` collection operators.
///
/// OpenSearch has no native "every element equals X" query, so both engines
/// emit a script. These strings must stay BYTE-IDENTICAL to the ones in
/// `src/tql/opensearch_components/query_converter.py`, including the
/// whitespace: the shared DSL fixture compares the emitted JSON exactly, which
/// is what makes a second copy in a second language safe to keep. Regenerate
/// from Python rather than retyping if either ever changes.
const ALL_SCRIPT: &str = "\n if (!doc.containsKey(params.field) || doc[params.field].size() == 0) {\n return false;\n }\n for (value in doc[params.field]) {\n if (value != params.value) {\n return false;\n }\n }\n return true;\n ";
const NOT_ALL_SCRIPT: &str = "\n // Check if field exists in the document mapping\n if (!doc.containsKey(params.field)) {\n // Field doesn't exist, so NOT ALL is true\n return true;\n }\n\n // Get field values\n def values = doc[params.field];\n\n // Empty array means not all elements are the value (vacuously true)\n if (values.size() == 0) {\n return true;\n }\n\n // Check if all elements match\n for (value in values) {\n if (value != params.value) {\n // Found an element that doesn't match\n return true;\n }\n }\n\n // All elements match, so NOT all is false\n return false;\n ";
impl QueryBuilder {
/// Create a new query builder
///
/// # Arguments
///
/// * `field_mappings` - Optional field mappings for intelligent query generation
pub fn new(field_mappings: Option<FieldMappings>) -> Self {
Self { field_mappings }
}
/// Convert AstValue to JsonValue
fn ast_value_to_json(value: &AstValue) -> JsonValue {
match value {
AstValue::String(s) => json!(s),
AstValue::Integer(i) => json!(i),
AstValue::Float(f) => json!(f),
AstValue::Boolean(b) => json!(b),
AstValue::List(list) => {
json!(list.iter().map(Self::ast_value_to_json).collect::<Vec<_>>())
}
AstValue::Null => json!(null),
}
}
/// Build an OpenSearch query from a TQL AST
///
/// # Arguments
///
/// * `ast` - The TQL abstract syntax tree
///
/// # Returns
///
/// OpenSearch Query DSL as JSON
///
/// # Example
///
/// ```ignore
/// use tql::parser::TqlParser;
/// use tql::opensearch::QueryBuilder;
///
/// let parser = TqlParser::new();
/// let ast = parser.parse("age > 25 AND status eq 'active'").unwrap();
/// let builder = QueryBuilder::new(None);
/// let query = builder.build_query(&ast).unwrap();
/// ```
pub fn build_query(&self, ast: &AstNode) -> Result<JsonValue> {
// Handle stats and query-with-stats AST nodes
match ast {
AstNode::StatsExpr(stats) => {
// Pure stats query (no filter): match_all + aggregations
let mut dsl = json!({ "query": { "match_all": {} }, "size": 0 });
let aggs =
super::stats_translator::translate_stats(stats, self.field_mappings.as_ref())
.map_err(OpenSearchError::TranslationError)?;
if let Some(aggs_obj) = aggs.get("aggs") {
dsl["aggs"] = aggs_obj.clone();
}
return Ok(dsl);
}
AstNode::QueryWithStats(qws) => {
// Filter + stats: build filter query + aggregations, size=0
let query_clause = self.build_query_clause(&qws.filter)?;
let mut dsl = json!({ "query": query_clause, "size": 0 });
let aggs = super::stats_translator::translate_stats(
&qws.stats,
self.field_mappings.as_ref(),
)
.map_err(OpenSearchError::TranslationError)?;
if let Some(aggs_obj) = aggs.get("aggs") {
dsl["aggs"] = aggs_obj.clone();
}
return Ok(dsl);
}
_ => {}
}
let query_clause = self.build_query_clause(ast)?;
Ok(json!({
"query": query_clause
}))
}
/// Does this comparison carry a VALUE to compare the mutated field against?
///
/// This is the one axis that gates the post-processing short-circuit below,
/// and it is deliberately a property of the NODE rather than a list of
/// operator names.
///
/// # The defect this fixes
///
/// `has_post_processing_mutators` used to be consulted unconditionally, at
/// the very top of `build_query_clause` — before the `not_exists` arm and
/// before `build_comparison`'s `is` / `is_not` / `eq null` / `ne null`
/// arms. Those arms are PRESENCE predicates: they ask whether the field has
/// a value at all, which no mutator changes, and their translations are
/// EXACT rather than approximate.
///
/// So forcing `exists` onto them did not over-broaden the phase-1 net the
/// way it does for a real comparison — it INVERTED it. Measured on this
/// branch, for each of the 24 non-collection entries in
/// `mutators::MUTATOR_NAMES`:
///
/// ```text
/// f not_exists {"bool":{"must_not":{"exists":{"field":"f"}}}}
/// f | md5 not_exists {"exists":{"field":"f"}} <- the complement
/// ```
///
/// Identically for `f not exists`, `f is null`, `f eq null` and `f = null`:
/// five spellings, each answering with precisely the documents that do NOT
/// match. `f exists`, `f is not null`, `f is_not null`, `f ne null` and
/// `f != null` — the other five of the ten — were unharmed only by the
/// coincidence that `exists` is already their correct answer, not because
/// anything protected them. (`is_not null` was missing from this list, which
/// left four named against a stated population of ten with five inverting.)
///
/// It returned HITS, not an error, which is the failure mode this whole
/// layer exists to remove. Python has gated on this axis since
/// `mutator_classification.blocks_pushdown` (`has_operand=`), and both
/// engines agree on every bare presence translation, so the Rust ordering
/// was the entire divergence.
///
/// # Why absence has two spellings here and one in Python
///
/// Python collapses both to `node["value"] is None`. Rust does not: a
/// valueless operator (`exists`, `not_exists`) parses to `value: None`,
/// while the `null` literal (`is null`, `is not null`, `eq null`) parses to
/// `value: Some(Value::Null)`. Both mean "no operand to compare against",
/// so both must answer `false` here. Matching only `None` fixes the
/// valueless spellings — `not exists` and `not_exists` — and leaves
/// `is null` / `eq null` / `= null` inverted, which is what makes this worth
/// a named function rather than an `is_some()` at the call site. Stated as
/// the two sets rather than as a count: this comment previously said "three
/// of the five" while naming three left inverted, three plus three over a
/// population of five, and no reader was going to add them up.
///
/// NOT an operator allow-list, deliberately. Python deleted the hand-written
/// ones in `mutator_analyzer.py` and `query_converter.py`; every one had
/// drifted from the others. What they ALL omitted is `matches`, `regexp`
/// and `cidr` — three, not five: they disagreed on `in`/`not_in` (the
/// analyzer's filtering list carried both, the converter's did not), which
/// is drift rather than a shared blind spot. Over the 24 mutators in
/// `NON_PUSHDOWN_MUTATORS` and the 61 operator spellings that remain once
/// the two presence predicates are removed from the 63 the live Python
/// grammar admits, **818 of those 1,464 combinations** pushed a clause onto
/// the raw field before the deletion and 0 do after. A name list here would
/// reintroduce that class in the engine that was already correct about it.
///
/// Both axes are named because the number is a product of two DERIVED sets
/// and means nothing without them. This comment said `373` until the sweep
/// of 2026-09-04: that figure came from `c3b316f`'s commit message, was
/// copied outward from there, and did not reproduce when re-measured.
/// `CHANGELOG.md` and `test_pushdown_parity.py` had already recorded that
/// it does not reproduce while this comment still asserted it as fact —
/// a Rust doc comment is not a place either of those two looks, which is
/// how a corrected number leaves its last copy behind.
/// The reproducible measurement and the
/// command that takes it are in `tests/unit/test_pushdown_parity.py`, above
/// `DERIVED_OPERATORS`; quote it from there rather than from here.
fn has_operand(comp: &ComparisonNode) -> bool {
!matches!(comp.value, None | Some(AstValue::Null))
}
/// Check if mutators require post-processing (cannot be pushed to OpenSearch)
fn has_post_processing_mutators(mutators: &Option<Vec<crate::parser::Mutator>>) -> bool {
mutators.as_ref().is_some_and(|mutators| {
mutators.iter().any(|m| {
let name = m.name.to_lowercase();
// These mutators require post-processing - they transform values
// and cannot be evaluated by OpenSearch
matches!(
name.as_str(),
"is_global"
| "is_private"
| "is_multicast"
| "is_loopback"
| "is_link_local"
| "nslookup"
| "geoip"
| "geoip_lookup"
| "geo"
| "lowercase"
| "uppercase"
| "trim"
| "length"
| "split"
| "replace"
| "b64encode"
| "b64decode"
| "urldecode"
| "hexencode"
| "hexdecode"
| "md5"
| "sha256"
| "refang"
| "defang"
)
})
})
}
fn build_query_clause(&self, node: &AstNode) -> Result<JsonValue> {
match node {
AstNode::Comparison(comp) => {
// Check if there are post-processing mutators on the field
// If so, we can only check that the field exists - the actual
// filtering will be done in post-processing
if Self::has_operand(comp)
&& Self::has_post_processing_mutators(&comp.field_mutators)
{
// For mutators that require post-processing, return exists query
// The actual filtering (is_global eq true, etc.) happens after
// results are fetched from OpenSearch
return Ok(json!({
"exists": {
"field": &comp.field
}
}));
}
// Handle "exists" operator with no value (field-only expressions like `field | nslookup`)
//
// There used to be a `has_enrichment_mutator` branch here that
// answered `match_all` for a `| nslookup` / `| geoip` chain. It
// was DEAD: `has_post_processing_mutators` immediately above
// lists every one of those names and returns `exists` first, so
// the branch could not be reached for any input that would have
// taken it. It also carried a FOURTH copy of the enrichment-alias
// list, and that copy was already wrong -- it omitted `geo`, the
// alias `create_mutator` accepts for Python and JS parity. Deleted
// rather than completed: a fourth list that agrees today is a
// fourth list that disagrees later, and the check above already
// makes the decision.
if comp.operator == "exists" && comp.value.is_none() {
return Ok(json!({
"exists": {
"field": &comp.field
}
}));
}
// `not_exists` carries no value, so without this arm it fell
// through to the "Comparison requires a value" error below and
// every query using it failed to translate — while the parser
// and the in-memory evaluator both accepted it, and Python
// translated it. Same shape as `cidr` in tql#198.
//
// It was invisible to the translation-coverage guard for a
// second reason: `exists`/`not_exists` are handled in
// `evaluator::evaluate_comparison` before dispatch, so they
// never appear in `comparator::compare`, which is where that
// guard derives its operator population from. The guard now
// derives the bypassing set from source as well.
//
// The field is used unresolved, matching the `exists` arm above
// and Python's `get_field_for_operator`: presence is a question
// about the field, not a subfield. A `.keyword` subfield
// carries `ignore_above`, so `exists` against it is silently
// false for longer values.
if comp.operator == "not_exists" && comp.value.is_none() {
return Ok(json!({
"bool": { "must_not": { "exists": { "field": &comp.field } } }
}));
}
let value = comp.value.as_ref().ok_or_else(|| {
OpenSearchError::TranslationError("Comparison requires a value".to_string())
})?;
self.build_comparison(&comp.field, &comp.operator, value)
}
AstNode::LogicalOp(logical) => {
self.build_logical(&logical.operator, &logical.left, &logical.right)
}
AstNode::UnaryOp(unary) => {
let inner = self.build_query_clause(&unary.operand)?;
Ok(json!({
"bool": {
"must_not": inner
}
}))
}
AstNode::MatchAll => Ok(json!({
"match_all": {}
})),
AstNode::CollectionOp(collection) => self.build_collection_op(collection),
AstNode::NslookupExpr(nslookup) => self.build_nslookup_expr(nslookup),
AstNode::GeoExpr(geo) => {
// Geo expressions require post-processing, similar to nslookup
// If there are conditions, use exists query on the field
// If no conditions, return match_all
if geo.conditions.is_some() {
Ok(json!({
"exists": {
"field": &geo.field
}
}))
} else {
Ok(json!({
"match_all": {}
}))
}
}
_ => Err(OpenSearchError::TranslationError(format!(
"Unsupported AST node type: {:?}",
node
))),
}
}
/// Escape a literal value for embedding in an OpenSearch `wildcard` pattern.
///
/// In a `wildcard` query `\` is the escape character and `*` / `?` are
/// metacharacters, so a raw value interpolated into a pattern gets
/// reinterpreted: `contains 'C:\Windows\Temp'` becomes the pattern
/// `*C:\Windows\Temp*`, which OpenSearch reads as `*C:WindowsTemp*` — a
/// guaranteed false negative on every Windows path. `contains 'a*b'`
/// likewise turns a literal asterisk into a wildcard.
///
/// Only the literal portion goes through here; the surrounding `*` the
/// operator contributes is added afterwards. `prefix` queries do not
/// interpret wildcards and must NOT be escaped.
fn escape_wildcard_value(value: &str) -> String {
value
.replace('\\', "\\\\")
.replace('*', "\\*")
.replace('?', "\\?")
}
/// The string form of a comparison operand, for the operators that embed it
/// in a `wildcard` pattern.
///
/// Every one of those arms used to read the operand with
/// `json_value.as_str().unwrap_or("")`, which returns `None` for anything
/// that is not a JSON string and DROPS IT. The empty string then went
/// straight into the pattern, so the operand vanished without a trace:
///
/// ```text
/// f contains true -> {"wildcard": {"f": "**"}} matches EVERYTHING
/// f contains 5 -> {"wildcard": {"f": "**"}} matches EVERYTHING
/// f startswith true -> {"wildcard": {"f": "*"}} matches EVERYTHING
/// f not contains 5 -> must_not(everything) matches NOTHING
/// ```
///
/// Numbers are the operationally important case, not booleans:
/// `event.code contains 46` is an ordinary thing to write, and it returned
/// the entire index. Measured on OpenSearch 2.19.4 against a keyword field
/// holding "5abc": `{"wildcard": {"f": "**"}}` -> 1 hit (the whole index),
/// `{"prefix": {"f": ""}}` -> 1 hit. Well-formed DSL, no error, wrong
/// documents.
///
/// A BOOLEAN renders `true` / `false`, not Python's `str(True)`.
///
/// This arm used to write `True`, on the reasoning that Python's translator
/// already did and that matching it beat inventing a third spelling. The
/// agreement was real; the spelling was wrong. Python's `str()` is the ONLY
/// thing in this stack that writes `True` — TQL's own boolean literals are
/// `true`, JSON writes `true`, and OpenSearch stores and returns `true` —
/// so both translators were building a pattern for a spelling nothing in
/// the index has. Measured on OpenSearch 2.19.4, a `keyword` field holding
/// `"true value"`:
///
/// ```text
/// wildcard *true* -> 1 hit
/// wildcard *True* -> 0 hits
/// ```
///
/// Zero hits and no error, which is indistinguishable from "nothing
/// matched". Settled by the product owner and changed on both sides
/// together.
///
/// A NULL renders `null`, not Python's `str(None)`, and it was settled the
/// same way one step later. It was left as `"None"` when the boolean case
/// was fixed, on the ground that both TRANSLATORS agreed — which was true,
/// and was the wrong comparison. Both EVALUATORS write `"null"`: a bareword
/// `null` operand reaches a string comparator as the four characters, here
/// via `ast_value_to_string` and in Python via the raw parsed string. So
/// the two execution paths for the same query selected different
/// documents. Measured on OpenSearch 2.19.4 over
/// `[{"f": "a null b"}, {"f": "a None b"}]`, both `keyword`:
///
/// ```text
/// f contains null evaluator (both engines) -> "a null b"
/// f contains null translator -> wildcard *None* -> "a None b"
/// ```
///
/// `null` is the JSON spelling, TQL's own literal, and what OpenSearch
/// stores and returns; `None` is Python's `str()` and nothing else in this
/// stack writes it.
///
/// A single-element list is unwrapped first, as Python does — the parser
/// hands one where a scalar is meant, and `as_str()` on the array was one
/// of the ways the operand disappeared.
///
/// KNOWN DIVERGENCE, deliberate and harmless: for a MULTI-element list this
/// emits serde's JSON (`["a","b"]`) where Python emits its own list repr
/// (`['a', 'b']`). A multi-element list is a nonsense operand to a substring
/// operator in both engines and matches nothing on either — unlike the
/// dropped operand above, which matched everything. Reproducing Python's
/// `repr` faithfully is not worth the fragility.
fn operand_text(value: &JsonValue) -> String {
match Self::unwrap_single(value.clone()) {
JsonValue::String(s) => s,
JsonValue::Number(n) => n.to_string(),
// `true`, NOT Python's `str(True)`. See the note above.
JsonValue::Bool(b) => if b { "true" } else { "false" }.to_string(),
JsonValue::Null => "null".to_string(),
other => other.to_string(),
}
}
/// The escaped string form, which is what every `wildcard` arm actually
/// wants. Split from [`Self::operand_text`] only so the two steps are
/// named; they are never used apart.
fn wildcard_pattern_operand(value: &JsonValue) -> String {
Self::escape_wildcard_value(&Self::operand_text(value))
}
/// Translate a PCRE pattern into a Lucene `regexp` clause.
///
/// `matches` has SEARCH semantics: an unanchored pattern is wrapped in `.*`
/// so it matches anywhere in the value, which is what the in-memory
/// evaluator has always done and what anyone writing a PCRE pattern
/// expects. `^` and `$` still mean full-match — `to_lucene_regex`
/// translates them rather than wrapping — so an author who anchored
/// deliberately keeps what they asked for.
///
/// This was previously unanchored on both sides, on the reasoning that
/// wrapping would change what every currently-working rule matches. The
/// measurement that settled it: of 188 regex patterns in the shipped
/// detection corpus, 5 are explicitly anchored and 151 are bare PCRE search
/// patterns. Lucene anchors implicitly, so every one of those matched the
/// WHOLE field value only — on a command line, URL or path field they
/// matched nothing, while the identical query searched correctly in memory.
///
/// Must stay in lockstep with `_regexp_query` in
/// `src/tql/opensearch_components/query_converter.py`; the shared
/// `regex_translation` fixture compares the two.
fn regexp_query(query_field: &str, value: &JsonValue) -> Result<JsonValue> {
// Unwrap single-element lists, as string operators do elsewhere.
let value = match value {
JsonValue::Array(arr) if arr.len() == 1 => &arr[0],
other => other,
};
// A non-string operand is RENDERED as text and translated like any
// other pattern. It used to be passed through untouched, on the stated
// reasoning that "mangling a non-string would be worse than letting
// OpenSearch report the type error". OpenSearch reports no such error:
// measured on 2.19.4 against a `keyword` field holding "true value",
//
// {"regexp": {"f": true}} -> 0 hits, NO ERROR
// {"regexp": {"f": ".*true.*"}} -> 1 hit
//
// because the bool is coerced to the string "true" and Lucene's regexp
// engine anchors implicitly, so it must equal the whole value. The
// pass-through therefore produced a silent zero, which is the outcome
// this translator exists to avoid — and `{"regexp": {"f": null}}` is a
// hard 400 ("value cannot be null") that takes every unrelated clause
// in the query down with it.
//
// `f matches 5` means the same thing as `f matches '5'`, so rendering
// and translating is also what the author wrote.
let owned = Self::operand_text(value);
let pattern = owned.as_str();
let translated = to_lucene_regex(pattern, true)?;
// flags NONE disables Lucene's optional operators — `~` complement, `&`
// intersection, `#` empty, `@` anystring, `<n-m>` interval — so those
// characters are literals. A pattern written for PCRE means the
// characters; left enabled, a stray `<` fails the whole query with
// "expected '>'".
let mut body = json!({ "value": translated.pattern, "flags": "NONE" });
if translated.case_insensitive {
body["case_insensitive"] = json!(true);
}
Ok(json!({ "regexp": { query_field: body } }))
}
/// The clause for a membership test against an EMPTY list.
///
/// `f in []` must match nothing (no value is a member of the empty set) and
/// `f not in []` must match everything (an empty exclusion excludes
/// nothing). The obvious rendering of the non-empty path -- a `bool.should`
/// of one `term` per value -- collapses to
/// `{"bool": {"should": [], "minimum_should_match": 1}}`, which reads as
/// "at least one of zero alternatives", i.e. impossible.
///
/// OpenSearch does not read it that way. It IGNORES `minimum_should_match`
/// when `should` is empty and treats the bool as `match_all`, so the query
/// means the exact OPPOSITE of its shape -- in BOTH directions. Measured
/// against the live cluster: the empty `should` matched 7,942,814 documents
/// and `must_not` of it matched 0.
///
/// `not_in []` is the operationally dangerous half. An empty exclusion list
/// is what a rule template produces when its exclusion set is empty, and
/// "excludes nothing" (correct) versus "excludes everything" (a rule that
/// silently stops firing) are opposite failures that look identical in a
/// result count.
///
/// An empty `terms` does NOT invert -- `{"terms": {f: []}}` matches nothing
/// and its negation matches everything -- which is why `in_cs []`, already
/// rendered as `terms`, was correct all along. So the defect is the
/// `bool.should` RENDERING specifically, not empty lists in general, and
/// the fix is to converge the two shapes rather than to special-case the
/// operator. `tests/integration/test_empty_list_opensearch_semantics.py`
/// pins both cluster facts and goes red the day OpenSearch changes either.
///
/// The case-insensitivity that the non-empty `in` path carries is not lost
/// here: there are no values to compare case-insensitively.
fn empty_membership_clause(query_field: &str) -> JsonValue {
json!({ "terms": { query_field: [] } })
}
/// Unwrap a single-element array to its element, leaving everything else
/// untouched.
///
/// The parser can hand a one-value list where a scalar is meant. The Python
/// converter unwraps for `cidr`/`not_cidr` before building the term query
/// (query_converter.py:520-522, 611-613); matching that keeps a
/// `term: {field: ["10.0.0.0/8"]}` — which OpenSearch rejects on an
/// `ip` field — from being emitted here.
fn unwrap_single(value: JsonValue) -> JsonValue {
match value {
JsonValue::Array(mut items) if items.len() == 1 => items.remove(0),
other => other,
}
}
/// The positive half of `eq` / `ne`, shared so the two cannot drift apart.
///
/// Three cases, mirroring `query_converter.py`'s `eq` branch exactly:
///
/// * mapped and ANALYZED — `match`. A `term` is not analyzed and matches
/// only if the whole value is one indexed token, so `message eq 'disk
/// full'` would find nothing.
/// * mapped and not analyzed — `term`, the exact match the user asked for.
/// * UNMAPPED — `match_phrase` for a string, `term` for anything else.
/// `match_phrase` is right on a text field and also on a keyword field
/// (the keyword analyzer emits the whole value as one token), so it is
/// the safe answer when the mapping is unknown. `term` is right only on
/// the keyword half, and guessing wrong returns zero hits rather than an
/// error. Non-strings keep `term`: analysis does not apply to numbers,
/// booleans or dates.
fn equality_clause(
query_field: &str,
json_value: &JsonValue,
resolved_is_analyzed_text: bool,
field_is_mapped: bool,
) -> JsonValue {
if field_is_mapped {
if resolved_is_analyzed_text {
return json!({ "match": { query_field: json_value } });
}
return json!({ "term": { query_field: json_value } });
}
if json_value.is_string() {
return json!({ "match_phrase": { query_field: json_value } });
}
json!({ "term": { query_field: json_value } })
}
/// Can this field carry `case_insensitive` on a `term` query?
///
/// Only string types can. OpenSearch rejects the parameter outright on an
/// `ip` field — "[source.ip] field which is of type [ip], does not support
/// case insensitive term queries" — and it is meaningless on numerics,
/// dates and booleans. Found by the Python live integration suite the
/// moment `in` became case-insensitive.
fn supports_case_insensitive_term(&self, field: &str) -> bool {
// Type the field this query will ACTUALLY target, subfield and all.
//
// This used to strip `.keyword` and type the BASE field, to work around
// `get_field_type` returning None for every subfield path. The
// workaround is wrong wherever the base and the subfield differ in
// type, which is the whole point of a multifield: on
// `{"type":"ip","fields":{"keyword":{"type":"keyword"}}}` resolution
// returns `f.keyword` while this guard typed `f` as `ip` and refused
// case-insensitivity. `in` is contractually case-insensitive
// (public/tql/user-guide/operators-reference.md), so `role in ['ADMIN']`
// silently missed `admin` on that shape while Python matched it —
// wrong answers, no error, and only on a multifield.
//
// `resolved_field_type` walks into subfields, so the guard now asks the
// only question that matters: can the field being queried carry
// `case_insensitive`?
match self
.field_mappings
.as_ref()
.and_then(|m| m.resolved_field_type(field))
{
// `wildcard` is a string type and accepts the parameter — verified
// on OpenSearch 2.19.4: a `term` with `case_insensitive` against a
// `{"type":"wildcard"}` field holding "Hello World" matches the
// lower-case spelling.
Some(FieldType::Keyword) | Some(FieldType::Text) | Some(FieldType::Wildcard) => true,
// Unmapped: do NOT assume string. Emitting `case_insensitive` for
// a field we cannot type makes OpenSearch answer HTTP 400 on an ip
// or numeric field and fail the whole search. A case-sensitive
// match on an unmapped string field is the narrower wrong answer,
// and it is what shipped before this change — so the fallback is a
// no-op rather than a regression.
None => false,
_ => false,
}
}
fn build_comparison(&self, field: &str, operator: &str, value: &AstValue) -> Result<JsonValue> {
// Convert AstValue to JsonValue
let json_value = Self::ast_value_to_json(value);
// Determine the actual field name to use (may include .keyword suffix)
// Propagates TypeError / UnsupportedOperation instead of silently
// falling back to the base field. An impossible operator/field pairing
// now fails loudly here rather than emitting a query that OpenSearch
// answers with zero hits — see `FieldMappings::get_query_field`.
let query_field = match self.field_mappings.as_ref() {
Some(m) => m.get_query_field(field, operator)?,
None => field.to_string(),
};
// A `term` query is not analyzed, so against an ANALYZED field it
// matches only if the whole value happens to be one token. Python emits
// `match` in that case (query_converter.py's eq branch, guarded on
// `should_use_term_query`), so `message eq 'disk full'` finds documents
// on Python and nothing on Rust. Resolve the same way here.
//
// This is only reachable for a text field with no keyword subfield —
// anything else resolved to a keyword form above.
let ci = self.supports_case_insensitive_term(&query_field);
let resolved_is_analyzed_text = self
.field_mappings
.as_ref()
.and_then(|m| m.resolved_field_type(&query_field))
.map(|t| *t == FieldType::Text)
.unwrap_or(false);
// Is there a mapping for this field AT ALL? Asked of the field the user
// wrote, not of `query_field`, which may already carry a `.keyword`
// suffix this resolution added.
//
// Absence is a THIRD case here, not a missing value that defaults to
// "keyword". Python branches on it explicitly
// (`field_name in self.intelligent_mappings or field_name in
// self.simple_mappings`) and emits `match_phrase` for a string against
// an unmapped field, because `match_phrase` is correct on BOTH a text
// and a keyword field while `term` is correct only on keyword. Rust
// emitted `term` unconditionally, so any query against an index whose
// mappings could not be fetched — or against a field the mapping does
// not name — silently missed every analyzed field: `term` is not
// analyzed, so it matches only when the whole value happens to be one
// token. Zero hits, no error.
let field_is_mapped = self
.field_mappings
.as_ref()
.is_some_and(|m| m.get_field_type(field).is_some());
// `eq`/`ne` against the bareword `null` are null PREDICATES, not term
// matches — see the `is` / `is_not` arms below for why. Handled before
// the operator match so no field-type branch can reach `term: null`.
if matches!(value, AstValue::Null) {
match operator {
"eq" | "=" => {
return Ok(json!({
"bool": { "must_not": { "exists": { "field": field } } }
}))
}
"ne" | "!=" => return Ok(json!({ "exists": { "field": field } })),
_ => {}
}
}
match operator {
"eq" => Ok(Self::equality_clause(
&query_field,
&json_value,
resolved_is_analyzed_text,
field_is_mapped,
)),
"ne" => Ok(json!({
"bool": {
"must_not": Self::equality_clause(
&query_field,
&json_value,
resolved_is_analyzed_text,
field_is_mapped,
)
}
})),
// OpenSearch accepts CIDR notation directly in a `term` query on an
// `ip`-typed field ("192.168.0.0/24" matches the whole subnet); no
// plugin or script is needed. This mirrors the Python converter
// (src/tql/opensearch_components/query_converter.py:519, 610), which
// has always had these arms.
//
// Without them `cidr` fell through to the catch-all below and every
// query using it failed with "Unsupported operator: cidr" — while the
// parser (parser/mod.rs) and the in-memory evaluator
// (comparator.rs:117) both accepted it. A shipped detection rule uses
// `cidr`, so on an agent (which runs THIS implementation) it could
// never execute. See #198.
"cidr" => Ok(json!({
"term": {
query_field: Self::unwrap_single(json_value)
}
})),
"not_cidr" => Ok(json!({
"bool": {
"must_not": {
"term": {
query_field: Self::unwrap_single(json_value)
}
}
}
})),
"gt" => Ok(json!({
"range": {
query_field: {
"gt": json_value
}
}
})),
"gte" => Ok(json!({
"range": {
query_field: {
"gte": json_value
}
}
})),
"lt" => Ok(json!({
"range": {
query_field: {
"lt": json_value
}
}
})),
"lte" => Ok(json!({
"range": {
query_field: {
"lte": json_value
}
}
})),
"contains" => {
// The value keeps its case and `case_insensitive` carries the
// matching intent. This replaced a `to_lowercase()` that was
// applied whenever the BASE field was text -- which is still
// true after get_query_field redirects to `.keyword`, so the
// lowercased value would have been matched against a
// case-preserving field and silently missed every mixed-case
// document. Also closes a parity gap: Python has always emitted
// `case_insensitive` here (tql#169).
Ok(json!({
"wildcard": {
query_field: {
"value": format!("*{}*", Self::wildcard_pattern_operand(&json_value)),
"case_insensitive": true
}
}
}))
}
"startswith" => {
// `wildcard` with a trailing `*`, NOT `prefix`.
//
// The two are semantically equivalent on a keyword field and
// `prefix` is the cheaper of the pair — but Python emits
// `wildcard` (query_converter.py's startswith branch), and this
// engine must emit the same DSL for the same query. A
// difference here means a rule validated through the Python
// package and a rule executed by the agent are not the same
// query, which is the class of divergence this file is being
// corrected for. If `prefix` is wanted for its performance,
// both implementations move together.
//
// The value is escaped because `wildcard` DOES interpret `*`
// and `?`; `prefix` did not, which is why the previous arm
// passed it raw.
Ok(json!({
"wildcard": {
query_field: {
"value": format!("{}*", Self::wildcard_pattern_operand(&json_value)),
"case_insensitive": true
}
}
}))
}
"endswith" => Ok(json!({
"wildcard": {
query_field: {
"value": format!("*{}", Self::wildcard_pattern_operand(&json_value)),
"case_insensitive": true
}
}
})),
"matches" => Self::regexp_query(&query_field, &json_value),
// Case-sensitive `in`. A `terms` query on a keyword field is
// inherently case-sensitive, which is exactly the intended
// semantics — this is the plain-`in` translation WITHOUT the
// `case_insensitive` flag the contract requires there.
//
// These were allow-listed as untranslatable under the belief that
// "Python raises TQLUnsupportedOperationError for these too, so
// Rust refusing them is PARITY". That premise was false when
// checked against HEAD: Python translates both, and has an explicit
// `in_cs` arm in `query_converter.py`. So the allow-list was
// documenting a Rust-only gap as a shared design decision.
"in_cs" => {
if let JsonValue::Array(arr) = &json_value {
Ok(json!({ "terms": { query_field: arr } }))
} else {
Ok(json!({ "term": { query_field: json_value } }))
}
}
"not_in_cs" => {
let inner = if let JsonValue::Array(arr) = &json_value {
json!({ "terms": { query_field: arr } })
} else {
json!({ "term": { query_field: json_value } })
};
Ok(json!({ "bool": { "must_not": inner } }))
}
"in" => {
// Use terms query
let values = if let JsonValue::Array(arr) = &json_value {
arr.clone()
} else {
vec![json_value.clone()]
};
if values.is_empty() {
return Ok(Self::empty_membership_clause(&query_field));
}
Ok(json!({
"bool": {
"should": values
.iter()
.map(|v| if ci {
json!({ "term": { query_field.clone(): {
"value": v, "case_insensitive": true
}}})
} else {
json!({ "term": { query_field.clone(): v } })
})
.collect::<Vec<_>>(),
"minimum_should_match": 1
}
}))
}
"between" => {
// Use range query with gte and lte
if let JsonValue::Array(arr) = &json_value {
if arr.len() == 2 {
Ok(json!({
"range": {
query_field: {
"gte": arr[0],
"lte": arr[1]
}
}
}))
} else {
Err(OpenSearchError::TranslationError(
"between operator requires array of 2 values".to_string(),
))
}
} else {
Err(OpenSearchError::TranslationError(
"between operator requires array value".to_string(),
))
}
}
// ---- negations -------------------------------------------------
//
// Each mirrors its positive form wrapped in `bool.must_not`, byte
// for byte with the Python converter. They were absent entirely:
// every one returned "Unsupported operator" while Python
// translated it, so a rule using `not contains` executed on the
// backend and refused to translate on the agent.
"not_in" => {
let values = match &json_value {
JsonValue::Array(a) => a.clone(),
other => vec![other.clone()],
};
if values.is_empty() {
return Ok(json!({
"bool": { "must_not": Self::empty_membership_clause(&query_field) }
}));
}
Ok(json!({
"bool": { "must_not": { "bool": {
"should": values
.iter()
.map(|v| if ci {
json!({ "term": { query_field.clone(): {
"value": v, "case_insensitive": true
}}})
} else {
json!({ "term": { query_field.clone(): v } })
})
.collect::<Vec<_>>(),
"minimum_should_match": 1
}}}
}))
}
"not_contains" => Ok(json!({
"bool": { "must_not": { "wildcard": { query_field: {
"value": format!("*{}*", Self::wildcard_pattern_operand(&json_value)),
"case_insensitive": true
}}}}
})),
"not_startswith" => Ok(json!({
"bool": { "must_not": { "wildcard": { query_field: {
"value": format!("{}*", Self::wildcard_pattern_operand(&json_value)),
"case_insensitive": true
}}}}
})),
"not_endswith" => Ok(json!({
"bool": { "must_not": { "wildcard": { query_field: {
"value": format!("*{}", Self::wildcard_pattern_operand(&json_value)),
"case_insensitive": true
}}}}
})),
// The negated arm MUST go through `regexp_query`, exactly as the
// positive `matches` arm does. It used to hand-build the `regexp`
// clause from `json_value.as_str()`, which skipped `to_lucene_regex`
// entirely -- and skipping it is silent in all four of its jobs:
//
// * PCRE anchors survive as literal characters. A Lucene `regexp`
// is implicitly whole-string-anchored, so `^Hel.*` demanded a
// literal `^`, matched NOTHING, and `must_not(nothing)` matched
// EVERY document. Measured against a 7-document index:
// `tags not matches '^Hel.*'` returned all 7 where 4 are right.
// * The search-semantics `.*` wrapping was not applied, so an
// unanchored pattern was whole-value-anchored instead.
// * An inline `(?i)` was not lifted to `case_insensitive`, so it
// was matched as the four literal characters.
// * A one-element list was not unwrapped -- `as_str()` returned
// None on the array, so the pattern became the EMPTY STRING and
// the clause negated "matches nothing at all".
// * PCRE shorthand (`\d`, `\w`, ...) reached Lucene untranslated.
//
// None of these is REJECTED, which is what made them dangerous.
// Measured on OpenSearch 2.19.4 against a `keyword` field holding
// "Hello", `flags: "NONE"`: `regexp (?i)hello` and
// `regexp Hell(?:o)` are both ACCEPTED and match nothing, while the
// translated `.*Hell[A-Za-z0-9_].*` matches. There is no 400 to
// notice -- `must_not` simply turns each silent zero into "every
// document in the index".
//
// Python has always routed both arms through `_regexp_query`, so
// this was a one-sided defect: the same rule answered differently
// depending on whether the backend or the agent built the query.
// `matches`/`regexp` is 190 clauses across the shipped detection
// corpus, so every negated one of those was affected.
"not_matches" => Ok(json!({
"bool": { "must_not": Self::regexp_query(&query_field, &json_value)? }
})),
"not_between" => {
if let JsonValue::Array(arr) = &json_value {
if arr.len() == 2 {
return Ok(json!({
"bool": { "must_not": { "range": { query_field: {
"gte": arr[0], "lte": arr[1]
}}}}
}));
}
}
Err(OpenSearchError::TranslationError(
"not_between operator requires array of 2 values".to_string(),
))
}
// ---- case-SENSITIVE variants -------------------------------------
//
// Same shapes as the default operators but WITHOUT
// `case_insensitive`, which is the entire difference. Note Python
// emits a bare value here rather than the object form.
"contains_cs" => Ok(json!({
"wildcard": { query_field: format!("*{}*", Self::wildcard_pattern_operand(&json_value)) }
})),
"not_contains_cs" => Ok(json!({
"bool": { "must_not": { "wildcard": {
query_field: format!("*{}*", Self::wildcard_pattern_operand(&json_value))
}}}
})),
"startswith_cs" => {
// `prefix`, not `wildcard`: this is what Python emits, and
// `prefix` does not interpret wildcards so the value is raw.
// `prefix` takes the RAW value, matching Python byte for byte:
// `f startswith_cs 5` emits `{"prefix": {"f": 5}}` there, and
// OpenSearch accepts it (measured on 2.19.4 against a keyword
// field holding "5abc": 1 hit, same as `{"prefix": {"f": "5"}}`).
// `as_str().unwrap_or("")` turned every non-string operand into
// the EMPTY prefix, which matches every document.
Ok(json!({ "prefix": { query_field: Self::unwrap_single(json_value.clone()) } }))
}
"not_startswith_cs" => Ok(json!({
"bool": { "must_not": { "prefix": {
query_field: Self::unwrap_single(json_value.clone())
}}}
})),
"endswith_cs" => Ok(json!({
"wildcard": { query_field: format!("*{}", Self::wildcard_pattern_operand(&json_value)) }
})),
"not_endswith_cs" => Ok(json!({
"bool": { "must_not": { "wildcard": {
query_field: format!("*{}", Self::wildcard_pattern_operand(&json_value))
}}}
})),
// ---- case-INSENSITIVE equality -----------------------------------
//
// A `wildcard` with no metacharacters and `case_insensitive: true`
// — a case-folded exact match. `term` cannot express that.
"eq_ci" => {
// Python splits on `isinstance(value, str)`: a string becomes
// the case-folded `wildcard`, and anything else becomes a plain
// `term` carrying the value unchanged — case does not exist for
// a number, boolean or date, so a term is already the right
// answer there. Rust ran every value through `as_str()`, so a
// non-string operand collapsed to `{"wildcard": {"f": ""}}` —
// measured: 0 hits, i.e. the clause silently stopped matching.
//
// Note Python does NOT unwrap a single-element list here (it
// emits `{"term": {"f": ["a"]}}`), unlike the substring arms.
// Mirrored rather than corrected: the two implementations must
// emit the same DSL, and the unwrapping asymmetry is Python's to
// change.
match &json_value {
JsonValue::String(raw) => Ok(json!({
"wildcard": { query_field: {
"value": Self::escape_wildcard_value(raw),
"case_insensitive": true
}}
})),
other => Ok(json!({ "term": { query_field: other } })),
}
}
// ---- null predicates ---------------------------------------------
//
// `eq null` / `ne null` translate exactly like `is null` /
// `is not null`, because OpenSearch does not index JSON nulls and
// therefore cannot tell a present-null from an absent field.
//
// Both engines previously emitted `{"term": {"f": null}}` here,
// which OpenSearch REJECTS outright — "field name is null or
// empty", HTTP 400 — so `f eq null` could not run against a cluster
// at all, in either implementation. That is a shared defect a
// Rust-vs-Python differential cannot see: the two agreed, and both
// were unrunnable. It took executing the emitted DSL to find it.
//
// The in-memory evaluators are stricter than this DSL can be: they
// distinguish present-null (matches `eq null`) from absent (does
// not). `is null` has carried that same unavoidable gap since it
// existed — the difference is a limit of the index, not of the
// translation.
// `query_field`, not the raw `field`. These arms used the raw name,
// which happened to be right — `get_query_field` now classifies
// `is`/`is_not` as existence operators and returns the base name
// anyway (audit finding F30) — but relying on the arm to bypass the
// resolver meant the classifier could say anything and nothing
// would notice. Routing through it makes the classification
// load-bearing and keeps this engine structurally identical to the
// Python converter, which does resolve here and got F30 wrong
// BECAUSE it did.
"is" => Ok(json!({
"bool": { "must_not": { "exists": { "field": query_field } } }
})),
"is_not" => Ok(json!({ "exists": { "field": query_field } })),
_ => Err(OpenSearchError::TranslationError(format!(
"Unsupported operator: {}",
operator
))),
}
}
fn build_logical(&self, operator: &str, left: &AstNode, right: &AstNode) -> Result<JsonValue> {
let left_clause = self.build_query_clause(left)?;
let right_clause = self.build_query_clause(right)?;
match operator.to_lowercase().as_str() {
"and" => Ok(json!({
"bool": {
"must": [left_clause, right_clause]
}
})),
"or" => Ok(json!({
"bool": {
"should": [left_clause, right_clause],
"minimum_should_match": 1
}
})),
_ => Err(OpenSearchError::TranslationError(format!(
"Unsupported logical operator: {}",
operator
))),
}
}
fn build_collection_op(&self, collection: &CollectionOpNode) -> Result<JsonValue> {
let json_value = Self::ast_value_to_json(&collection.value);
let operator = collection.operator.to_lowercase();
let comparison_op = &collection.comparison_operator;
// `f any ['a']` parses with a single-element LIST as its value, but the
// clause is about one element. Rust emitted `{"term": {"f": ["a"]}}` —
// a term query whose value is an array, which is not the same question
// and which OpenSearch does not answer usefully — while Python unwrapped
// and emitted `{"term": {"f": "a"}}`. Unwrap once, in one place, so
// every collection operator below inherits it.
let scalar_value = Self::unwrap_single(json_value.clone());
// Resolve the field exactly as `build_comparison` does. This function
// used `collection.field` RAW, so it bypassed field resolution
// entirely: on a text+keyword multifield Rust emitted
// `{"term": {"f": "a"}}` against the ANALYZED field while Python
// emitted `{"term": {"f.keyword": "a"}}`. A term query on an analyzed
// field matches only when the whole value is one token — zero hits, no
// error, and only on the mapping shape that is most common in ECS.
let query_field = match self.field_mappings.as_ref() {
Some(m) => m.get_query_field(&collection.field, &operator)?,
None => collection.field.clone(),
};
// `all` / `not_all` are answered by a Painless script that ignores the
// inner comparison clause entirely, so they are dispatched here rather
// than through the quantifier wrapper below. They already emit valid
// DSL for a list value (the script compares each element to
// `params.value`, so a list never matches -- which is what both
// evaluators answer), and are deliberately unchanged.
if operator == "all" {
return Self::all_script(&query_field, &json_value, ALL_SCRIPT);
}
if operator == "not_all" {
return Self::all_script(&query_field, &json_value, NOT_ALL_SCRIPT);
}
// A value that is STILL an array after `unwrap_single` is a LIST on the
// right-hand side of `any` / `none`. That is ILL-TYPED, and the decision
// taken is to say so rather than to answer it.
//
// `in` is already the membership operator. `f any ['a','b']` does not
// ask "is any element of `f` one of these two values" -- it asks whether
// any single ELEMENT of `f` equals the two-element LIST, which no scalar
// element can. The docs describe a scalar operand throughout; the
// grammar admits a list here only because `value` includes `list_value`.
//
// The history is the argument for refusing rather than answering:
//
// * It first emitted `{"term": {"f": ["a","b"]}}`, which OpenSearch
// answers with HTTP 400 `[term] query does not support array of
// values` -- loud, but it failed the WHOLE search, taking every
// unrelated clause in the query down with it.
// * It was then changed to `match_none` (and `must_not: match_none`
// for `none`), which states the in-memory answer directly and
// executes cleanly -- but silently. `f none ['a','b']` is an
// EXCLUSION clause that then excludes nothing, with no signal at all
// that the operator was misused.
//
// Neither is what a query language should do with a query it can tell is
// wrong. Blast radius of refusing is zero: no shipped rule uses the list
// form, and no user can be holding a working saved query in it, because
// for its entire life it either returned HTTP 400 or zero hits.
//
// The refusal covers every inner comparison operator, including `ne`.
// `ne` used to be special-cased to `exists`, on the reasoning that every
// element DIFFERS from a list so the clause is true whenever the field
// has any value at all. That is a coherent answer to an incoherent
// question -- and answering it lets `f any != ['a','b']` go on looking
// like a working query. What is wrong is the operand, not the
// comparison.
//
// The in-memory evaluator still answers `false` here rather than
// raising; `f9_coll_any_multi` pins that and is unchanged. Translator
// and evaluator therefore differ in KIND (a refusal versus a `false`)
// while agreeing that nothing matches. Closing that gap means changing
// the evaluator, which is a separate decision.
if scalar_value.is_array() {
return Err(OpenSearchError::TypeError {
field: collection.field.clone(),
field_type: "list operand".to_string(),
operator: operator.clone(),
suggestion: format!(
" `{op}` takes a single value and tests it against each \
element of the field; it does not take a list. Use `in` for \
membership: `{field} in [...]` (or `{field} not in [...]` \
to exclude).",
op = operator,
field = collection.field,
),
});
}
// Build the inner comparison query
let inner_query = match comparison_op.as_str() {
"eq" => json!({
"term": {
query_field.clone(): scalar_value
}
}),
"ne" => json!({
"bool": {
"must_not": {
"term": {
query_field.clone(): scalar_value
}
}
}
}),
"gt" => json!({
"range": {
query_field.clone(): {
"gt": scalar_value
}
}
}),
"gte" => json!({
"range": {
query_field.clone(): {
"gte": scalar_value
}
}
}),
"lt" => json!({
"range": {
query_field.clone(): {
"lt": scalar_value
}
}
}),
"lte" => json!({
"range": {
query_field.clone(): {
"lte": scalar_value
}
}
}),
// `wildcard_pattern_operand`, not `as_str().unwrap_or("")`: a
// non-string operand used to be dropped here too, leaving `**`,
// which matches every document.
"contains" => json!({
"wildcard": {
collection.field.clone(): format!(
"*{}*",
Self::wildcard_pattern_operand(&scalar_value)
)
}
}),
_ => {
return Err(OpenSearchError::TranslationError(format!(
"Unsupported collection comparison operator: {}",
comparison_op
)));
}
};
Self::wrap_collection_operator(&operator, inner_query)
}
/// Wrap an inner clause in the collection operator's quantifier.
///
/// Extracted so the array-valued arm above and the ordinary scalar arm
/// below cannot drift: they must compose identically, or `none` and `any`
/// stop being complements for exactly one shape of value.
fn wrap_collection_operator(operator: &str, inner_query: JsonValue) -> Result<JsonValue> {
match operator {
"any" => {
// ANY: At least one element matches (OpenSearch handles arrays automatically)
Ok(inner_query)
}
// `all` / `not_all` never reach here -- dispatched to `all_script`
// above, because their script form ignores the inner clause.
"none" | "not_any" => {
// NONE / NOT ANY: no element matches. The two are the same
// question, and Python collapses them at parse time.
Ok(json!({
"bool": {
"must_not": inner_query
}
}))
}
// NOT NONE is the double negative of NONE, i.e. ANY. Python
// normalises `not none` to `any` in the parser and never sees this
// operator; Rust keeps the spelling, so it is collapsed here
// instead. Both emit the same DSL.
"not_none" => Ok(inner_query),
_ => Err(OpenSearchError::TranslationError(format!(
"Unsupported collection operator: {}",
operator
))),
}
}
/// A `script` query for the `all` / `not_all` collection operators.
///
/// The field name is interpolated into a Painless script parameter, so it
/// is validated first — the mirror of `_validate_script_field_name` in the
/// Python converter. A field name is attacker-influenced whenever a query
/// is built from user input, and an unchecked one reaches a script context.
fn all_script(field: &str, json_value: &JsonValue, source: &str) -> Result<JsonValue> {
if !Self::is_safe_script_field_name(field) {
return Err(OpenSearchError::TranslationError(format!(
"Field name '{field}' contains characters not allowed in script context"
)));
}
Ok(json!({
"script": {
"script": {
"source": source,
"params": { "field": field, "value": Self::unwrap_single(json_value.clone()) }
}
}
}))
}
/// Mirror of `_SAFE_FIELD_NAME_RE` in the Python converter:
/// `^[a-zA-Z_@][a-zA-Z0-9_.@\-]*$`. Written out rather than pulled in as a
/// regex dependency, and kept in the same shape so the two can be compared
/// by eye.
fn is_safe_script_field_name(field: &str) -> bool {
let mut chars = field.chars();
match chars.next() {
Some(c) if c.is_ascii_alphabetic() || c == '_' || c == '@' => {}
_ => return false,
}
chars.all(|c| c.is_ascii_alphanumeric() || matches!(c, '_' | '.' | '@' | '-'))
}
fn build_nslookup_expr(&self, nslookup: &NslookupExprNode) -> Result<JsonValue> {
// Nslookup expressions are enrichment operations that require post-processing
// If there are conditions (filters on the nslookup results), use exists query
// If no conditions (just enrichment), return match_all to fetch all documents
if nslookup.conditions.is_some() {
Ok(json!({
"exists": {
"field": &nslookup.field
}
}))
} else {
Ok(json!({
"match_all": {}
}))
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::parser::TqlParser;
/// `field | nslookup` translates to `exists`, for EVERY enrichment alias.
///
/// `has_post_processing_mutators` lists all four names and returns first, so
/// the `has_enrichment_mutator` branch that used to sit below it — carrying a
/// FOURTH copy of the alias list, already missing `geo` — was unreachable.
/// This pins the answer the reachable path actually gives, including for the
/// alias the dead branch omitted, so deleting it is a no-op that stays one.
#[test]
fn every_enrichment_alias_translates_to_exists() {
let parser = TqlParser::new();
let builder = QueryBuilder::new(None);
for alias in ["nslookup", "geoip", "geoip_lookup", "geo"] {
let query = format!("hostname | {alias}");
let ast = parser.parse(&query).expect("parse failed");
let dsl = builder.build_query(&ast).expect("translation failed");
assert_eq!(
dsl["query"]["exists"]["field"], "hostname",
"`{query}` translated to {dsl}"
);
}
}
/// `status` has no mapping here (`QueryBuilder::new(None)`), so this is the
/// UNMAPPED path and `match_phrase` is the correct answer — right on a text
/// field and on a keyword field alike.
///
/// This asserted `term` until the unmapped-field fix. It was pinning the
/// defect: `term` is not analyzed, so it silently returned zero hits for
/// any multi-token value on an analyzed field, and Python has always
/// emitted `match_phrase` here. Both engines are checked against each other
/// case-by-case in the shared `dsl_translation` fixture
/// (`unmapped__*`); this unit test stays as the fast local signal.
#[test]
fn test_simple_equality() {
let parser = TqlParser::new();
let ast = parser.parse("status eq 'active'").unwrap();
let builder = QueryBuilder::new(None);
let query = builder.build_query(&ast).unwrap();
assert_eq!(
query,
json!({
"query": {
"match_phrase": {
"status": "active"
}
}
})
);
}
#[test]
fn test_range_query() {
let parser = TqlParser::new();
let ast = parser.parse("age > 25").unwrap();
let builder = QueryBuilder::new(None);
let query = builder.build_query(&ast).unwrap();
assert_eq!(
query,
json!({
"query": {
"range": {
"age": {
"gt": 25
}
}
}
})
);
}
#[test]
fn test_and_query() {
let parser = TqlParser::new();
let ast = parser.parse("age > 25 AND status eq 'active'").unwrap();
let builder = QueryBuilder::new(None);
let query = builder.build_query(&ast).unwrap();
// Verify it's a bool/must query
assert!(query["query"]["bool"]["must"].is_array());
assert_eq!(query["query"]["bool"]["must"].as_array().unwrap().len(), 2);
}
#[test]
fn test_or_query() {
let parser = TqlParser::new();
let ast = parser
.parse("status eq 'active' OR status eq 'pending'")
.unwrap();
let builder = QueryBuilder::new(None);
let query = builder.build_query(&ast).unwrap();
// Verify it's a bool/should query
assert!(query["query"]["bool"]["should"].is_array());
assert_eq!(
query["query"]["bool"]["should"].as_array().unwrap().len(),
2
);
assert_eq!(query["query"]["bool"]["minimum_should_match"], 1);
}
#[test]
fn test_not_query() {
let parser = TqlParser::new();
let ast = parser.parse("NOT (age < 18)").unwrap();
let builder = QueryBuilder::new(None);
let query = builder.build_query(&ast).unwrap();
// Verify it's a bool/must_not query
assert!(query["query"]["bool"]["must_not"]["range"].is_object());
}
#[test]
fn test_detection_rule_with_is_global_and_nslookup() {
let parser = TqlParser::new();
let query_str =
"event.code = 3 AND destination.ip | is_global eq true AND destination.ip | nslookup";
let ast = parser.parse(query_str).unwrap();
// Print the AST for debugging
eprintln!("AST for detection rule query:\n{:#?}", ast);
let builder = QueryBuilder::new(None);
let query = builder.build_query(&ast).unwrap();
// Print the generated DSL
eprintln!(
"Generated OpenSearch DSL:\n{}",
serde_json::to_string_pretty(&query).unwrap()
);
// The query should have a bool with must clauses
assert!(
query["query"]["bool"]["must"].is_array(),
"Expected bool/must query, got: {}",
serde_json::to_string_pretty(&query).unwrap()
);
let must_clauses = query["query"]["bool"]["must"].as_array().unwrap();
// Should have 3 clauses: event.code=3, is_global, nslookup
// Actually, the structure depends on how AND is parsed
eprintln!("Number of must clauses: {}", must_clauses.len());
}
// ---------------------------------------------------------------------
// wildcard-value escaping
// ---------------------------------------------------------------------
/// Pull the pattern text out of a `{"wildcard": {field: pattern}}` clause.
fn wildcard_pattern(query: &str) -> String {
let parser = TqlParser::new();
let ast = parser.parse(query).expect("query should parse");
let builder = QueryBuilder::new(None);
let dsl = builder.build_query(&ast).expect("query should translate");
let clause = dsl["query"]
.get("wildcard")
.unwrap_or_else(|| panic!("expected a wildcard clause, got {dsl}"));
let payload = clause
.as_object()
.and_then(|o| o.values().next())
.expect("wildcard clause should carry one field");
match payload {
JsonValue::String(s) => s.clone(),
JsonValue::Object(o) => o["value"]
.as_str()
.expect("value should be a string")
.to_string(),
other => panic!("unexpected wildcard payload: {other}"),
}
}
/// Decode a wildcard pattern the way OpenSearch does: a backslash escapes
/// the next character; unescaped `*` / `?` are metacharacters. Asserting on
/// the decoded text makes these tests semantic rather than brittle
/// substring checks — it is what OpenSearch will actually search for.
fn opensearch_unescape(pattern: &str) -> String {
let mut out = String::new();
let mut chars = pattern.chars();
while let Some(c) = chars.next() {
if c == '\\' {
if let Some(next) = chars.next() {
out.push(next);
}
} else if c != '*' && c != '?' {
out.push(c);
}
}
out
}
#[test]
fn escape_wildcard_value_escapes_the_metacharacters() {
assert_eq!(QueryBuilder::escape_wildcard_value(r"\"), r"\\");
assert_eq!(QueryBuilder::escape_wildcard_value("*"), r"\*");
assert_eq!(QueryBuilder::escape_wildcard_value("?"), r"\?");
assert_eq!(QueryBuilder::escape_wildcard_value("plain"), "plain");
}
#[test]
fn escape_order_does_not_double_escape_inserted_backslashes() {
// `\` must be escaped before `*`, or the backslash inserted in front of
// an escaped `*` would itself get escaped.
assert_eq!(QueryBuilder::escape_wildcard_value(r"\*"), r"\\\*");
}
/// REGRESSION: `contains 'C:\Windows\Temp'` emitted the pattern
/// `*C:\Windows\Temp*`, which OpenSearch reads as `*C:WindowsTemp*` — the
/// separators are consumed as escape characters and the query matches
/// nothing. Windows paths dominate the detection corpus.
#[test]
fn contains_survives_opensearch_pattern_decoding() {
let decoded = opensearch_unescape(&wildcard_pattern(
r"process.command_line contains 'C:\\Windows\\Temp'",
));
assert_eq!(decoded, r"C:\Windows\Temp");
}
#[test]
fn endswith_survives_opensearch_pattern_decoding() {
let decoded = opensearch_unescape(&wildcard_pattern(
r"process.executable endswith '\\cmd.exe'",
));
assert_eq!(decoded, r"\cmd.exe");
}
#[test]
fn literal_asterisk_is_escaped_not_promoted_to_a_wildcard() {
assert_eq!(wildcard_pattern("path contains 'a*b'"), r"*a\*b*");
}
#[test]
fn values_without_special_characters_are_unchanged() {
assert_eq!(wildcard_pattern("user.name contains 'alice'"), "*alice*");
assert_eq!(wildcard_pattern("user.name endswith 'ice'"), "*ice");
}
// ---------------------------------------------------------------------
// regexp emission
//
// The translation rules themselves live in `crate::regex_compat` and are
// pinned by a fixture both languages run. What these tests own is the
// WIRING: that the builder calls the translator at all, and that the
// emitted clause carries `flags: NONE` and the lifted case-insensitivity.
// ---------------------------------------------------------------------
/// Pull the body out of a `{"regexp": {field: ...}}` clause.
fn regexp_body(query: &str) -> JsonValue {
let parser = TqlParser::new();
let ast = parser.parse(query).expect("query should parse");
let builder = QueryBuilder::new(None);
let dsl = builder.build_query(&ast).expect("query should translate");
let clause = dsl["query"]
.get("regexp")
.unwrap_or_else(|| panic!("expected a regexp clause, got {dsl}"));
clause
.as_object()
.and_then(|o| o.values().next())
.expect("regexp clause should carry one field")
.clone()
}
/// REGRESSION: the builder emitted `{"regexp": {field: pattern}}` with the
/// pattern untouched. A rule using `\d` produced a query OpenSearch rejects
/// at SEARCH time — it loads, validates, schedules, and throws on every run
/// while looking healthy.
#[test]
fn regexp_patterns_are_translated_for_lucene() {
assert_eq!(
regexp_body(r"process.command_line regexp '\d+'")["value"],
// `.*` on both ends: `matches` SEARCHES. Lucene anchors implicitly,
// so an unanchored PCRE pattern must be wrapped or it matches only
// a whole field value -- 151 of the 188 patterns in the shipped
// detection corpus are bare search patterns like this one, and
// every one of them matched nothing on a command-line field.
r".*[0-9]+.*"
);
assert_eq!(
regexp_body(r"process.command_line regexp '(?:foo|bar)'")["value"],
".*(foo|bar).*"
);
}
/// flags NONE makes `~ & # @ <n-m>` literals. A PCRE author writing `<`
/// means the character; left enabled it is the interval operator and fails
/// the whole query with "expected '>'".
#[test]
fn regexp_queries_disable_lucene_optional_operators() {
let body = regexp_body("process.command_line regexp 'a<b'");
assert_eq!(body["flags"], "NONE");
assert_eq!(body["value"], ".*a<b.*");
}
#[test]
fn regexp_queries_carry_lifted_case_insensitivity() {
let body = regexp_body("process.command_line regexp '(?i)abc'");
assert_eq!(body["value"], ".*abc.*");
assert_eq!(body["case_insensitive"], true);
}
#[test]
fn regexp_queries_omit_case_insensitivity_when_not_requested() {
let body = regexp_body("process.command_line regexp 'abc'");
assert!(
body.get("case_insensitive").is_none(),
"case_insensitive should be absent, got {body}"
);
}
/// A pattern Lucene cannot execute is refused at BUILD time rather than
/// shipped as a query that throws on every run.
#[test]
fn untranslatable_patterns_are_refused_at_build_time() {
let parser = TqlParser::new();
let ast = parser
.parse(r"process.command_line regexp 'foo\bbar'")
.expect("query should parse");
let err = QueryBuilder::new(None)
.build_query(&ast)
.expect_err("a word boundary has no Lucene equivalent");
assert!(
err.to_string().contains("word boundary"),
"unexpected error: {err}"
);
}
/// The escaped-backslash path form must NOT be mistaken for `\b`.
#[test]
fn windows_path_with_escaped_backslash_still_builds() {
let body = regexp_body(r"process.executable regexp 'C:\\\\BUnzip\\\\Setup\.exe'");
assert_eq!(body["value"], r".*C:\\BUnzip\\Setup\.exe.*");
}
}