// TQL Grammar Definition
// Matches Python pyparsing implementation
WHITESPACE = _{ " " | "\t" | "\r" | "\n" }
COMMENT = _{ "//" ~ (!"\n" ~ ANY)* }
// ========== Basic Tokens ==========
// Identifiers (field names, function names, etc.)
// Supports @timestamp and similar Elasticsearch/OpenSearch standard fields
// Rules:
// - May start with @ (like @timestamp, @metadata)
// - First char after optional @ must be a letter or underscore
// - Can contain letters, numbers, underscores, dots, hyphens
// - @ only allowed at start (time@stamp is INVALID)
// Hyphens are allowed in the continuation of an identifier (e.g., event-code,
// user-agent, x-forwarded-for). This matches real-world field names in ECS and
// HTTP headers. Note that a bare hyphen followed by a digit could be ambiguous
// with negative numbers; the pest PEG resolves this by preferring the
// longest match within the identifier rule.
identifier = @{ "@"? ~ (ASCII_ALPHA | "_") ~ (ASCII_ALPHANUMERIC | "_" | "." | "-")* }
// String literals (single or double quoted)
string_double = ${ "\"" ~ inner_double ~ "\"" }
inner_double = @{ (!("\"" | "\\") ~ ANY | "\\" ~ ANY)* }
string_single = ${ "'" ~ inner_single ~ "'" }
inner_single = @{ (!("'" | "\\") ~ ANY | "\\" ~ ANY)* }
string = { string_double | string_single }
// Numeric literals
integer = @{ "-"? ~ ASCII_DIGIT+ }
float = @{ "-"? ~ ASCII_DIGIT+ ~ "." ~ ASCII_DIGIT+ ~ (^"e" ~ ("+" | "-")? ~ ASCII_DIGIT+)? }
number = { float | integer }
// Boolean literals
boolean = { ^"true" | ^"false" }
// Null literal
null = { ^"null" }
// ========== Operators ==========
// ---------- Word-operator boundary guard ----------
//
// EVERY operator spelled with letters needs this. Without it an operator matches
// the PREFIX of a longer word, and when the remainder is itself a valid unquoted
// identifier the query parses into a different, entirely plausible query:
//
// f containszq -> operator `contains`, value `zq`
// f eqzq -> operator `eq`, value `zq`
// f nonexists -> operator `none`, value `xists` -- the INVERTED answer
//
// 45 of the 70 operator spellings leaked this way; only in/any/all/none and their
// `not_` forms were guarded, because those were fixed one incident at a time. The
// guard is now applied to the class rather than to the symptom, and
// `tql/tests/operator_word_boundary.rs` DERIVES the operator list from this file
// so a new word operator added without a guard fails a test rather than shipping.
//
// Two mechanical rules, both learned the hard way:
//
// 1. A rule carrying the lookahead must be ATOMIC (`@`). As a plain rule the
// trailing lookahead lets pest's implicit WHITESPACE into the span;
// `normalize_operator` maps ' ' to '_' and produced `not_in_`, which no
// evaluator knows -- it parsed cleanly and died at EVALUATION time.
// 2. Only alternatives ENDING in a word character take the guard. `>=` and `=`
// legitimately abut their operand (`f >=1`), so guarding them would reject
// valid queries.
word_end = _{ !(ASCII_ALPHANUMERIC | "_") }
// `not` as a negating prefix. Guarded, so `f notcontains 'x'` is rejected as
// Python already rejects it, while `f not contains 'x'` and `f !contains 'x'`
// both still parse.
//
// `not_word` must be ATOMIC and separate. Inlined as `^"not" ~ word_end` inside
// the non-atomic `neg_prefix`, pest inserts implicit WHITESPACE between the
// literal and the lookahead -- so on `age not between 20 and 40` the guard was
// evaluated at the `b` of `between` and failed, rejecting a valid query. This is
// rule 1 above, arrived at from the other side: there the stray whitespace
// widened a SPAN, here it moved a LOOKAHEAD.
not_word = @{ ^"not" ~ word_end }
neg_prefix = _{ "!" | not_word }
// Comparison operators
eq_op = @{ ^"eq" ~ word_end | "=" }
ne_op = @{ ^"ne" ~ word_end | "!=" }
gt_op = @{ ^"gt" ~ word_end | ">" }
gte_op = @{ ^"gte" ~ word_end | ">=" }
lt_op = @{ ^"lt" ~ word_end | "<" }
lte_op = @{ ^"lte" ~ word_end | "<=" }
// String operators
contains_op = @{ ^"contains" ~ word_end }
startswith_op = @{ ^"startswith" ~ word_end }
endswith_op = @{ ^"endswith" ~ word_end }
matches_op = @{ (^"matches" | ^"regexp" | ^"regex") ~ word_end }
// Case-sensitive string operators (explicit)
contains_cs_op = @{ ^"contains_cs" ~ word_end }
startswith_cs_op = @{ ^"startswith_cs" ~ word_end }
endswith_cs_op = @{ ^"endswith_cs" ~ word_end }
// Case-insensitive equality operator (explicit)
eq_ci_op = @{ ^"eq_ci" ~ word_end }
// Negated string operators.
// The two-token form delegates its guard to the already-guarded base rule; the
// underscore form gets its own atomic `_word` rule. Same shape as `not_in_op`.
not_contains_word = @{ ^"not_contains" ~ word_end }
not_contains_op = { neg_prefix ~ contains_op | not_contains_word }
not_startswith_word = @{ ^"not_startswith" ~ word_end }
not_startswith_op = { neg_prefix ~ startswith_op | not_startswith_word }
not_endswith_word = @{ ^"not_endswith" ~ word_end }
not_endswith_op = { neg_prefix ~ endswith_op | not_endswith_word }
not_matches_word = @{ (^"not_matches" | ^"not_regexp" | ^"not_regex") ~ word_end }
not_matches_op = { neg_prefix ~ matches_op | not_matches_word }
// Negated case-sensitive string operators
not_contains_cs_word = @{ ^"not_contains_cs" ~ word_end }
not_contains_cs_op = { neg_prefix ~ contains_cs_op | not_contains_cs_word }
not_startswith_cs_word = @{ ^"not_startswith_cs" ~ word_end }
not_startswith_cs_op = { neg_prefix ~ startswith_cs_op | not_startswith_cs_word }
not_endswith_cs_word = @{ ^"not_endswith_cs" ~ word_end }
not_endswith_cs_op = { neg_prefix ~ endswith_cs_op | not_endswith_cs_word }
// CIDR operators (for IP address matching)
cidr_op = @{ ^"cidr" ~ word_end }
not_cidr_word = @{ ^"not_cidr" ~ word_end }
not_cidr_op = { neg_prefix ~ cidr_op | not_cidr_word }
// Existence operators
exists_op = @{ ^"exists" ~ word_end }
not_exists_word = @{ ^"not_exists" ~ word_end }
not_exists_op = { neg_prefix ~ exists_op | not_exists_word }
// NULL operators
is_op = @{ ^"is" ~ word_end }
is_not_word = @{ ^"is_not" ~ word_end }
is_not_op = { is_op ~ not_word | is_not_word }
// Collection operators
// in_op uses word boundary to avoid matching prefix of in_cs
in_op = @{ ^"in" ~ !(ASCII_ALPHANUMERIC | "_") }
// The underscore form must be its own ATOMIC rule. Written inline as
// `^"not_in" ~ !(...)` it is a two-element sequence, so pest inserts implicit
// WHITESPACE between the literal and the lookahead and the rule's span swallows
// the following space. `normalize_operator` then maps ' ' to '_' and produces
// "not_in_", which no evaluator knows -- `role not_in ['ADMIN']` failed at
// EVALUATION time with "Unknown operator: not_in_" while parsing cleanly.
not_in_word = @{ ^"not_in" ~ !(ASCII_ALPHANUMERIC | "_") }
not_in_op = { neg_prefix ~ in_op | not_in_word }
// Case-sensitive collection operators
in_cs_op = @{ ^"in_cs" ~ word_end }
not_in_cs_word = @{ ^"not_in_cs" ~ word_end }
not_in_cs_op = { neg_prefix ~ in_cs_op | not_in_cs_word }
between_op = @{ ^"between" ~ word_end }
not_between_word = @{ ^"not_between" ~ word_end }
not_between_op = { neg_prefix ~ between_op | not_between_word }
// Array operators (for collection operations)
//
// These need the same word-boundary guard `in_op` carries, and for a sharper
// reason: `collection_comparison` accepts `collection_op ~ field ~ value`, so an
// unguarded `^"none"` matches the FIRST FOUR LETTERS OF A FIELD NAME. Every
// field beginning with any/all/none was mis-lexed -- `nonexistent_field not
// exists` split into the operator `none` plus the field `xistent_field` and died
// with "expected value", while `optional_field not exists` parsed fine. Atomic
// (`@`) matters too: as a plain rule the trailing lookahead lets pest's implicit
// WHITESPACE into the span, which is the "not_in_" bug above.
any_op = @{ ^"any" ~ !(ASCII_ALPHANUMERIC | "_") }
all_op = @{ ^"all" ~ !(ASCII_ALPHANUMERIC | "_") }
none_op = @{ ^"none" ~ !(ASCII_ALPHANUMERIC | "_") }
not_any_word = @{ ^"not_any" ~ !(ASCII_ALPHANUMERIC | "_") }
not_all_word = @{ ^"not_all" ~ !(ASCII_ALPHANUMERIC | "_") }
not_none_word = @{ ^"not_none" ~ !(ASCII_ALPHANUMERIC | "_") }
not_any_op = { neg_prefix ~ any_op | not_any_word }
not_all_op = { neg_prefix ~ all_op | not_all_word }
not_none_op = { neg_prefix ~ none_op | not_none_word }
// IMPORTANT: _cs variants must be listed before their base operators
// to prevent PEG greedy matching from consuming the prefix.
// For example, not_in_cs_op before not_in_op, in_cs_op before in_op, etc.
comparison_op = {
gte_op | lte_op | gt_op | lt_op | ne_op | eq_ci_op | eq_op |
not_startswith_cs_op | not_endswith_cs_op | not_contains_cs_op |
not_startswith_op | not_endswith_op | not_contains_op | not_matches_op |
startswith_cs_op | endswith_cs_op | contains_cs_op |
startswith_op | endswith_op | contains_op | matches_op |
not_cidr_op | cidr_op |
is_not_op | is_op |
not_exists_op | exists_op |
not_in_cs_op | not_in_op | in_cs_op | in_op |
not_between_op | between_op
}
// Logical operators
and_op = @{ ^"and" ~ word_end | "&&" }
or_op = @{ ^"or" ~ word_end | "||" }
not_op = @{ ^"not" ~ word_end | "!" }
logical_op = { and_op | or_op }
// ========== Type Hints ==========
type_hint = { "::" ~ type_name }
// Atomic + guarded for the same reason every word operator is: unguarded,
// `f::intzq` lexed as the hint `int` followed by the residue `zq`.
//
// THIS RULE IS LOAD-BEARING IN BOTH ENGINES. It was not: until 91849d2 `type_hint`
// was parsed, stored on the AST node, and read by nothing outside this module, so
// `f::int eq 'Hello'` and `f::boolean eq 'Hello'` both evaluated to `true`,
// identical to the un-hinted query. Every name below now reaches a decision in
// `field_accessor::apply_type_hint` -- a conversion that can refuse the value, or
// an entry in `STRUCTURAL_TYPE_HINTS` stating why there is nothing to convert.
//
// `date`, `geo`, `object` and `ip` were added HERE ONLY AFTER that landed, and
// the order was the point rather than a courtesy: adding a name to a layer that
// consumes no names converts a loud parse error into a silent no-op, which is the
// exact defect both engines were being repaired for. `ip` is the case that shows
// it -- Python carried a complete `ip` branch (format validation, CIDR handling
// for the `cidr` operator, its own error message) that no query could reach in
// either engine, because neither grammar spelled the name.
//
// The set here must equal `field_accessor::known_type_hints()` and Python's
// `KNOWN_TYPE_HINTS`. All three are asserted equal, from both sides, by
// `tql/tests/type_hint_evaluation_tests.rs` and
// `tests/unit/test_type_hint_coercion.py` -- each SCRAPES this file rather than
// transcribing it, because four independent transcriptions of this list is how
// they drifted.
type_name = @{
(^"string" | ^"str" |
^"integer" | ^"int" |
^"number" | ^"decimal" |
^"float" | ^"double" |
^"boolean" | ^"bool" |
^"list" | ^"array" |
^"date" | ^"geo" |
^"object" | ^"ip") ~ word_end
}
// ========== Mutators ==========
mutator_name = @{ (ASCII_ALPHA | "_") ~ (ASCII_ALPHANUMERIC | "_")* }
mutator_named_arg = { identifier ~ "=" ~ (string | number | boolean | null | identifier) }
mutator_arg = { mutator_named_arg | string | number | boolean | null | identifier }
mutator_args = { mutator_arg ~ ("," ~ mutator_arg)* }
// Mutator - must not be followed by "stats" keyword
// The trick: mutator_name won't match "stats" if we check it properly
mutator = { "|" ~ !^"stats" ~ mutator_name ~ ("(" ~ mutator_args? ~ ")")? }
// ========== Field Expressions ==========
field_name = { identifier }
// BOTH orders of type hint and mutator chain are accepted, deliberately:
//
// f::number | trim > 75 (hint first -- what Python has always spelled)
// f | trim::number > 75 (hint last -- what Rust has always spelled)
//
// Until now each engine accepted ONLY its own spelling and rejected the other's,
// so ANY query combining a mutator with a type hint was non-portable between the
// backend query API (Python) and the agent's detection engine (Rust). Two live
// corpora exist and each was written against one engine's rule, so converging on
// a single spelling would silently invalidate half of them. Accepting both
// invalidates neither.
//
// THE ORDER IS NOTATION, NOT PIPELINE ORDER. Both engines apply FIELD MUTATORS
// FIRST and the TYPE HINT SECOND, whichever way round the query spells them --
// `evaluator.rs` mutates at `field_mutators` before it reaches `apply_type_hint`,
// and `evaluator.py` does the same (`apply_mutators` then `apply_type_hint`).
// That was already true of each engine for its own spelling, so accepting the
// other spelling changes no result: `f | b64decode::number` and
// `f::number | b64decode` both decode first and coerce second. Held by
// `tests/unit/test_hint_mutator_order.py` and
// `tql/tests/hint_mutator_order_tests.rs`, which use a base64 value the hint
// CANNOT read until the mutator has run -- a test that goes red if either engine
// ever moves the hint to the front of the pipeline.
//
// The alternation, not `type_hint? ~ mutator* ~ type_hint?`, because the latter
// accepts TWO hints (`f::int | trim::str`) and there is no defensible answer for
// which one wins. Ordered choice makes the doubled spelling consume `f::int |
// trim` and strand `::str`, which fails the enclosing rule -- so it is rejected,
// as it is in Python.
field_with_mutators = { field_name ~ (type_hint ~ mutator* | mutator* ~ type_hint?) }
// ========== Value Expressions ==========
// An empty list is NOT a value. `list_value` carried `"[" ~ "]"` as its FIRST
// alternative, so Rust alone parsed `f in []` while Python's
// `DelimitedList(self.list_item)` (no `Optional`) rejected it and the JS
// intellisense followed the stricter engine. That divergence let the agent's
// Rust evaluator accept a query the backend query API and the editor both
// refuse -- and the query is one an author almost never means: an empty
// membership list is what a rule whose value list came from an empty variable
// produces, and it cannot express any intent worth guessing at. Refusing at
// parse is this campaign's settled answer for "cannot express what was
// intended", and it converges all three engines on the same answer for every
// list-taking operator (`in`, `not_in`, `in_cs`, `not_in_cs`, `any`, `all`,
// `none`, `between`), which is where Python already stood.
//
// Note this is a PARSE-level refusal on purpose. The DSL both translators now
// emit for an empty list is `{"terms":{"f":[]}}` (converged in af3ad07), which
// is no longer the match-everything clause it once was -- so the argument for
// refusing is the unexpressible intent, not the emitted DSL.
list_value = { "[" ~ value ~ ("," ~ value)* ~ "]" }
value_with_mutators = { value ~ mutator* }
// IP/CIDR literals (must come before number to avoid partial float match on dotted IPs)
cidr_value = @{ ASCII_DIGIT{1,3} ~ ("." ~ ASCII_DIGIT{1,3}){3} ~ "/" ~ ASCII_DIGIT{1,2} }
ip_value = @{ ASCII_DIGIT{1,3} ~ ("." ~ ASCII_DIGIT{1,3}){3} }
// Note: identifier must come after string/number/boolean/null to avoid ambiguity
// This matches Python's simple_value which includes identifiers for unquoted strings
// cidr_value and ip_value before number to prevent 192.168 being parsed as float
value = { list_value | string | cidr_value | ip_value | number | boolean | null | identifier }
// ========== Comparison Expressions ==========
// Unary operators (field exists, field not exists)
unary_comparison = {
field_with_mutators ~ (exists_op | not_exists_op)
}
// Binary operators (field op value)
binary_comparison = {
field_with_mutators ~ comparison_op ~ value_with_mutators
}
// IS NULL / IS NOT NULL
is_null_comparison = {
field_with_mutators ~ (is_not_op | is_op) ~ null
}
// IN operator, REVERSED: the value is on the left and one or more FIELDS on the
// right. Two spellings, and they are deliberately SEPARATE rules because they
// sit on opposite sides of `binary_comparison` in the `comparison` alternation
// below. Merging them back into one rule re-opens the defect described there.
//
// `'val' in arrayfield` -> in_field_comparison (unbracketed)
// `'val' in [f1, f2]` -> in_fields_comparison (bracketed)
in_fields_list = { "[" ~ field_name ~ ("," ~ field_name)* ~ "]" }
in_field_comparison = {
value_with_mutators ~ in_op ~ field_with_mutators
}
in_fields_comparison = {
value_with_mutators ~ in_op ~ in_fields_list
}
// BETWEEN operator: field between [value1, value2] or field between value1 and value2
between_comparison = {
field_with_mutators ~ (not_between_op | between_op) ~ (list_value | value ~ ^"and" ~ value)
}
// Collection operators: ANY field op value, ALL field op value
// Also supports field-first syntax: field ANY op value (Python style)
// comparison_op is optional (defaults to eq): field ANY value, ANY field value
collection_op = { not_any_op | not_all_op | not_none_op | any_op | all_op | none_op }
collection_comparison = {
collection_op ~ field_with_mutators ~ comparison_op ~ value_with_mutators |
collection_op ~ field_with_mutators ~ value_with_mutators |
field_with_mutators ~ collection_op ~ comparison_op ~ value_with_mutators |
field_with_mutators ~ collection_op ~ value_with_mutators
}
// Field-only expression (for selecting/transforming without filtering)
field_only_expression = {
field_with_mutators
}
// All comparison types
// ORDER IS LOAD-BEARING, and `in_fields_comparison` sitting AFTER
// `binary_comparison` is the load-bearing part.
//
// `in_fields_list` is a list of bare `field_name`s, and `value_with_mutators`
// accepts a bare `identifier` -- so `f in [true]`, `f in [g]` and
// `f in [true, false]` are ALL well-formed reversed-`in` expressions. While
// `in_fields_comparison` was tried first, PEG's ordered choice took that
// reading and never reached `binary_comparison`, which transposed the field and
// the value:
//
// f in [true] parsed as {field: "true", operator: "eq", value: "f"}
//
// -- a comparison against a field literally named `true`, emitting
// `{"match_phrase": {"true": "f"}}`. It returns ZERO HITS and NO ERROR, which is
// indistinguishable from "nothing matched". Python has never had this: its
// `field_in_values` alternative is listed ahead of `value_in_field_list`, so the
// forward reading wins there for every bracketed list. Rust now agrees.
//
// The UNBRACKETED reverse form (`'val' in arrayfield`) keeps its priority ahead
// of `binary_comparison`, because `binary_comparison` WOULD match it -- `f in g`
// means "g contains f" in both engines, and demoting this rule would silently
// flip it to the forward reading. That asymmetry is why the two spellings are
// separate rules.
comparison = {
collection_comparison |
between_comparison |
in_field_comparison |
is_null_comparison |
unary_comparison |
binary_comparison |
in_fields_comparison |
field_only_expression
}
// ========== Logical Expressions ==========
// Parenthesized expression
paren_expr = { "(" ~ logical_expr ~ ")" }
// Primary expression (comparison or parenthesized)
primary = { paren_expr | comparison }
// NOT expression (uses term to support double negation: NOT NOT x = 1)
not_expr = { not_op ~ term }
// Term (NOT expression or primary)
term = { not_expr | primary }
// AND/OR chain
logical_expr = { term ~ (logical_op ~ term)* }
// ========== Stats Expressions ==========
// Aggregation functions
//
// ATOMIC, with the same trailing word-boundary guard `in_op`/`any_op` carry
// above -- and for the same reason, arrived at from the other direction.
//
// pest's `|` is ORDERED CHOICE: it commits to the first alternative that
// matches and never backtracks into this rule when the enclosing
// `aggregation` sequence later fails at `"("`. Written as a bare alternation
// that put `percentile` ahead of `percentiles` and `p` ahead of `pct`, every
// name having a listed PREFIX was unreachable: `percentiles(n)` matched
// `percentile`, then demanded `(` and found `s`, and the whole query died as
// a syntax error. Eight aggregations could never be dispatched to: six
// shadowed -- `percentile` consumed `percentiles`, `percentile_rank` and
// `percentile_ranks`, and `p` consumed `pct`, `pct_rank` and `pct_ranks` --
// plus `cardinality` and `unique_count`, which had no entry at all. Python
// accepted all eight, so this was also the whole Rust half of the aggregation
// name-parity gap.
//
// What was on the OTHER side of the parser is deliberately not asserted here:
// this file cannot observe `stats_evaluator.rs` or `stats_translator.rs`, and a
// claim about their arms stood in this comment for a release while being wrong
// for four of the eight. `stats_agg_name_reachability_tests.rs` computes
// reachability instead of describing it.
//
// TWO separate things fix this, and it matters which does what, because a
// comment here previously said the guard alone makes the ORDER IRRELEVANT and
// that "adding a future alias in the wrong place cannot resurrect the bug".
// That is FALSE, and it is the dangerous direction: it invites a maintainer to
// alphabetise this rule. Measured by reordering the percentile family
// shortest-first at HEAD with the guard left in place -- `percentiles(n, 95)`,
// `pct(n, 95)` and `pct_ranks(n, 2)` all became parse errors again. pest does
// not backtrack into an ordered choice when a LATER element of the sequence
// fails, so the guard cannot rescue a shortest-first list.
//
// 1. THE ORDER IS LOAD-BEARING. Every prefix pair is listed longest-first
// (`percentile_ranks` before `percentile_rank` before `percentiles` before
// `percentile`; `pct_ranks` before `pct_rank` before `pct` before `p`;
// `unique_count` before `unique`; `standard_deviation` before `std`).
// Keep it that way.
// 2. The trailing guard does a DIFFERENT job: it stops a name matching the
// front of an unrelated longer identifier, so `sumx(n)` is rejected rather
// than read as `sum`.
//
// `stats_agg_name_reachability_tests.rs` is what makes a reorder fail loudly
// rather than silently -- it pins that every advertised name parses AND
// reaches its implementation, and that `sumx(n)` is still rejected. Verified:
// with the rule reordered shortest-first, its reachability tests fail.
//
// Atomic (`@`) is load-bearing, not stylistic: as a plain rule the trailing
// lookahead lets pest's implicit WHITESPACE into the span, which is the
// "not_in_" defect documented at the top of this file.
agg_func_name = @{
(
^"count" | ^"sum" | ^"average" | ^"avg" | ^"mean" |
^"min" | ^"max" | ^"median" | ^"med" |
^"standard_deviation" | ^"std" |
^"percentile_ranks" | ^"percentile_rank" | ^"percentiles" | ^"percentile" |
^"pct_ranks" | ^"pct_rank" | ^"pct" | ^"p" |
^"unique_count" | ^"cardinality" | ^"distinct" | ^"unique" | ^"values"
) ~ !(ASCII_ALPHANUMERIC | "_")
}
// Aggregation modifiers (top N, bottom N)
agg_modifier = { (^"top" | ^"bottom") ~ integer }
// Percentile values
percentile_values = { number ~ ("," ~ number)* }
// Aggregation with field
agg_field = { field_with_mutators | "*" }
// Full aggregation specification
// Percentile values can be inside parens: percentile(field, 50, 75) or outside: percentile(field) 50, 75
//
// The top-N modifier likewise has TWO spellings, and until now each engine
// accepted only its own:
//
// | stats sum(x) top 10 by y parsed in Rust, REJECTED by Python
// | stats sum(x, top 10) by y parsed in Python, REJECTED by Rust
//
// so stats top-N had NO portable spelling at all -- not a case of "prefer this
// one", but of no query text that both the backend query API and the agent's
// detection engine would accept. The parity warning in `tql/README.md` stated
// only one direction, which implied a portable alternative that did not exist.
// Both engines now accept both, and both build the SAME `modifier` + `limit`
// pair, so the two spellings select the same N buckets.
//
// The four-way ordered choice, rather than making both slots independently
// optional, is what refuses `sum(x, top 10) top 5`. With two independent
// optionals that spelling parses and `parse_aggregation` silently keeps whichever
// modifier it saw last -- a query carrying two contradictory limits that answers
// with one of them and reports nothing. Here the first alternative consumes
// `, top 10 )` and strands ` top 5`, which `stats_expr` cannot continue from, so
// the doubled spelling is refused. Python's `agg_function` uses the same
// four-way MatchFirst for the same reason.
aggregation = {
agg_func_name ~ "(" ~ (
(field_with_mutators ~ "," ~ agg_modifier ~ ")") |
(agg_field? ~ "," ~ percentile_values ~ ")") |
(agg_field? ~ ")" ~ agg_modifier) |
(agg_field? ~ ")")
) ~
(percentile_values)? ~
(^"as" ~ identifier)?
}
// Group by fields
group_by_field = { field_with_mutators ~ (^"top" ~ integer)? }
group_by_list = { group_by_field ~ ("," ~ group_by_field)* }
// Visualization hint with optional parameters
viz_value = { string | number | boolean | identifier }
viz_param = { identifier ~ "=" ~ viz_value }
viz_params = { "(" ~ viz_param ~ ("," ~ viz_param)* ~ ")" }
// The CLOSED set of chart types, mirroring `self.viz_types` in
// `src/tql/parser_components/grammar.py`. This rule used to be a bare
// `identifier`, so `=> myfancychart` was well-formed to this engine and a
// TQLSyntaxError to Python -- and Python is what the backend query API runs, so
// a query the agent's engine accepted was refused the moment a user saved it.
//
// Converged towards the STRICTER engine deliberately: a closed set is what lets
// a typo be reported at parse time instead of reaching a renderer that has no
// such chart. Longest-first within each shared prefix, because PEG alternation
// is ordered and `bar` would otherwise shadow `barchart`.
viz_type = @{
(^"barchart" | ^"bar") ~ !(ASCII_ALPHANUMERIC | "_") |
(^"horizontal_bar" | ^"grouped_bar" | ^"stacked_bar") ~ !(ASCII_ALPHANUMERIC | "_") |
^"line" ~ !(ASCII_ALPHANUMERIC | "_") |
(^"stacked_area" | ^"area") ~ !(ASCII_ALPHANUMERIC | "_") |
(^"nested_pie" | ^"pie") ~ !(ASCII_ALPHANUMERIC | "_") |
(^"nested_donut" | ^"donut") ~ !(ASCII_ALPHANUMERIC | "_") |
^"scatter" ~ !(ASCII_ALPHANUMERIC | "_") |
^"heatmap" ~ !(ASCII_ALPHANUMERIC | "_") |
^"treemap" ~ !(ASCII_ALPHANUMERIC | "_") |
^"sunburst" ~ !(ASCII_ALPHANUMERIC | "_") |
^"table" ~ !(ASCII_ALPHANUMERIC | "_") |
^"number" ~ !(ASCII_ALPHANUMERIC | "_") |
^"gauge" ~ !(ASCII_ALPHANUMERIC | "_") |
^"map" ~ !(ASCII_ALPHANUMERIC | "_") |
^"chord" ~ !(ASCII_ALPHANUMERIC | "_") |
^"auto" ~ !(ASCII_ALPHANUMERIC | "_")
}
viz_hint = { "=>" ~ viz_type ~ viz_params? }
// Stats expression (pipe prefix is optional for standalone stats)
stats_expr = {
"|"? ~ ^"stats" ~
aggregation ~ ("," ~ aggregation)* ~
(^"by" ~ group_by_list)? ~
viz_hint?
}
// ========== Top-Level Query ==========
// Combined query with stats
// The mutator rule now prevents consuming | stats, so no lookahead needed
query_with_stats = { logical_expr ~ stats_expr }
// Full query
query = {
SOI ~
(query_with_stats | stats_expr | logical_expr) ~
EOI
}