// TQL Grammar Definition
// Matches Python pyparsing implementation
WHITESPACE = _{ " " | "\t" | "\r" | "\n" }
COMMENT = _{ "//" ~ (!"\n" ~ ANY)* }
// ========== Basic Tokens ==========
// Identifiers (field names, function names, etc.)
// Supports @timestamp and similar Elasticsearch/OpenSearch standard fields
// Rules:
// - May start with @ (like @timestamp, @metadata)
// - First char after optional @ must be a letter or underscore
// - Can contain letters, numbers, underscores, dots, hyphens
// - @ only allowed at start (time@stamp is INVALID)
// Hyphens are allowed in the continuation of an identifier (e.g., event-code,
// user-agent, x-forwarded-for). This matches real-world field names in ECS and
// HTTP headers. Note that a bare hyphen followed by a digit could be ambiguous
// with negative numbers; the pest PEG resolves this by preferring the
// longest match within the identifier rule.
identifier = @{ "@"? ~ (ASCII_ALPHA | "_") ~ (ASCII_ALPHANUMERIC | "_" | "." | "-")* }
// String literals (single or double quoted)
string_double = ${ "\"" ~ inner_double ~ "\"" }
inner_double = @{ (!("\"" | "\\") ~ ANY | "\\" ~ ANY)* }
string_single = ${ "'" ~ inner_single ~ "'" }
inner_single = @{ (!("'" | "\\") ~ ANY | "\\" ~ ANY)* }
string = { string_double | string_single }
// Numeric literals
integer = @{ "-"? ~ ASCII_DIGIT+ }
float = @{ "-"? ~ ASCII_DIGIT+ ~ "." ~ ASCII_DIGIT+ ~ (^"e" ~ ("+" | "-")? ~ ASCII_DIGIT+)? }
number = { float | integer }
// Boolean literals
boolean = { ^"true" | ^"false" }
// Null literal
null = { ^"null" }
// ========== Operators ==========
// Comparison operators
eq_op = { ^"eq" | "=" }
ne_op = { ^"ne" | "!=" }
gt_op = { ^"gt" | ">" }
gte_op = { ^"gte" | ">=" }
lt_op = { ^"lt" | "<" }
lte_op = { ^"lte" | "<=" }
// String operators
contains_op = { ^"contains" }
startswith_op = { ^"startswith" }
endswith_op = { ^"endswith" }
matches_op = { ^"matches" | ^"regexp" | ^"regex" }
// Case-sensitive string operators (explicit)
contains_cs_op = { ^"contains_cs" }
startswith_cs_op = { ^"startswith_cs" }
endswith_cs_op = { ^"endswith_cs" }
// Case-insensitive equality operator (explicit)
eq_ci_op = { ^"eq_ci" }
// Negated string operators
not_contains_op = { ("!" | ^"not") ~ ^"contains" | ^"not_contains" }
not_startswith_op = { ("!" | ^"not") ~ ^"startswith" | ^"not_startswith" }
not_endswith_op = { ("!" | ^"not") ~ ^"endswith" | ^"not_endswith" }
not_matches_op = { ("!" | ^"not") ~ (^"matches" | ^"regexp" | ^"regex") | ^"not_matches" | ^"not_regexp" | ^"not_regex" }
// Negated case-sensitive string operators
not_contains_cs_op = { ("!" | ^"not") ~ ^"contains_cs" | ^"not_contains_cs" }
not_startswith_cs_op = { ("!" | ^"not") ~ ^"startswith_cs" | ^"not_startswith_cs" }
not_endswith_cs_op = { ("!" | ^"not") ~ ^"endswith_cs" | ^"not_endswith_cs" }
// CIDR operators (for IP address matching)
cidr_op = { ^"cidr" }
not_cidr_op = { ("!" | ^"not") ~ ^"cidr" | ^"not_cidr" }
// Existence operators
exists_op = { ^"exists" }
not_exists_op = { "!" ~ ^"exists" | ^"not" ~ ^"exists" | ^"not_exists" }
// NULL operators
is_op = { ^"is" }
is_not_op = { ^"is" ~ ^"not" | ^"is_not" }
// Collection operators
// in_op uses word boundary to avoid matching prefix of in_cs
in_op = @{ ^"in" ~ !(ASCII_ALPHANUMERIC | "_") }
not_in_op = { ("!" | ^"not") ~ in_op | ^"not_in" ~ !(ASCII_ALPHANUMERIC | "_") }
// Case-sensitive collection operators
in_cs_op = { ^"in_cs" }
not_in_cs_op = { ("!" | ^"not") ~ ^"in_cs" | ^"not_in_cs" }
between_op = { ^"between" }
not_between_op = { ("!" | ^"not") ~ ^"between" | ^"not_between" }
// Array operators (for collection operations)
any_op = { ^"any" }
all_op = { ^"all" }
none_op = { ^"none" }
not_any_op = { ("!" | ^"not") ~ ^"any" | ^"not_any" }
not_all_op = { ("!" | ^"not") ~ ^"all" | ^"not_all" }
not_none_op = { ("!" | ^"not") ~ ^"none" | ^"not_none" }
// IMPORTANT: _cs variants must be listed before their base operators
// to prevent PEG greedy matching from consuming the prefix.
// For example, not_in_cs_op before not_in_op, in_cs_op before in_op, etc.
comparison_op = {
gte_op | lte_op | gt_op | lt_op | ne_op | eq_ci_op | eq_op |
not_startswith_cs_op | not_endswith_cs_op | not_contains_cs_op |
not_startswith_op | not_endswith_op | not_contains_op | not_matches_op |
startswith_cs_op | endswith_cs_op | contains_cs_op |
startswith_op | endswith_op | contains_op | matches_op |
not_cidr_op | cidr_op |
is_not_op | is_op |
not_exists_op | exists_op |
not_in_cs_op | not_in_op | in_cs_op | in_op |
not_between_op | between_op
}
// Logical operators
and_op = { ^"and" | "&&" }
or_op = { ^"or" | "||" }
not_op = { ^"not" | "!" }
logical_op = { and_op | or_op }
// ========== Type Hints ==========
type_hint = { "::" ~ type_name }
type_name = {
^"string" | ^"str" |
^"integer" | ^"int" |
^"number" | ^"decimal" |
^"float" | ^"double" |
^"boolean" | ^"bool" |
^"list" | ^"array"
}
// ========== Mutators ==========
mutator_name = @{ (ASCII_ALPHA | "_") ~ (ASCII_ALPHANUMERIC | "_")* }
mutator_named_arg = { identifier ~ "=" ~ (string | number | boolean | null | identifier) }
mutator_arg = { mutator_named_arg | string | number | boolean | null | identifier }
mutator_args = { mutator_arg ~ ("," ~ mutator_arg)* }
// Mutator - must not be followed by "stats" keyword
// The trick: mutator_name won't match "stats" if we check it properly
mutator = { "|" ~ !^"stats" ~ mutator_name ~ ("(" ~ mutator_args? ~ ")")? }
// ========== Field Expressions ==========
field_name = { identifier }
field_with_mutators = { field_name ~ mutator* ~ type_hint? }
// ========== Value Expressions ==========
list_value = { "[" ~ "]" | "[" ~ value ~ ("," ~ value)* ~ "]" }
value_with_mutators = { value ~ mutator* }
// IP/CIDR literals (must come before number to avoid partial float match on dotted IPs)
cidr_value = @{ ASCII_DIGIT{1,3} ~ ("." ~ ASCII_DIGIT{1,3}){3} ~ "/" ~ ASCII_DIGIT{1,2} }
ip_value = @{ ASCII_DIGIT{1,3} ~ ("." ~ ASCII_DIGIT{1,3}){3} }
// Note: identifier must come after string/number/boolean/null to avoid ambiguity
// This matches Python's simple_value which includes identifiers for unquoted strings
// cidr_value and ip_value before number to prevent 192.168 being parsed as float
value = { list_value | string | cidr_value | ip_value | number | boolean | null | identifier }
// ========== Comparison Expressions ==========
// Unary operators (field exists, field not exists)
unary_comparison = {
field_with_mutators ~ (exists_op | not_exists_op)
}
// Binary operators (field op value)
binary_comparison = {
field_with_mutators ~ comparison_op ~ value_with_mutators
}
// IS NULL / IS NOT NULL
is_null_comparison = {
field_with_mutators ~ (is_not_op | is_op) ~ null
}
// IN operator with multiple fields: value in [field1, field2, ...]
// Also supports: value in field (checks if value exists in array field)
in_fields_list = { "[" ~ field_name ~ ("," ~ field_name)* ~ "]" }
in_fields_comparison = {
value_with_mutators ~ in_op ~ (in_fields_list | field_with_mutators)
}
// BETWEEN operator: field between [value1, value2] or field between value1 and value2
between_comparison = {
field_with_mutators ~ (not_between_op | between_op) ~ (list_value | value ~ ^"and" ~ value)
}
// Collection operators: ANY field op value, ALL field op value
// Also supports field-first syntax: field ANY op value (Python style)
// comparison_op is optional (defaults to eq): field ANY value, ANY field value
collection_op = { not_any_op | not_all_op | not_none_op | any_op | all_op | none_op }
collection_comparison = {
collection_op ~ field_with_mutators ~ comparison_op ~ value_with_mutators |
collection_op ~ field_with_mutators ~ value_with_mutators |
field_with_mutators ~ collection_op ~ comparison_op ~ value_with_mutators |
field_with_mutators ~ collection_op ~ value_with_mutators
}
// Field-only expression (for selecting/transforming without filtering)
field_only_expression = {
field_with_mutators
}
// All comparison types
comparison = {
collection_comparison |
between_comparison |
in_fields_comparison |
is_null_comparison |
unary_comparison |
binary_comparison |
field_only_expression
}
// ========== Logical Expressions ==========
// Parenthesized expression
paren_expr = { "(" ~ logical_expr ~ ")" }
// Primary expression (comparison or parenthesized)
primary = { paren_expr | comparison }
// NOT expression (uses term to support double negation: NOT NOT x = 1)
not_expr = { not_op ~ term }
// Term (NOT expression or primary)
term = { not_expr | primary }
// AND/OR chain
logical_expr = { term ~ (logical_op ~ term)* }
// ========== Stats Expressions ==========
// Aggregation functions
agg_func_name = {
^"count" | ^"sum" | ^"avg" | ^"average" | ^"mean" |
^"min" | ^"max" | ^"median" | ^"med" |
^"std" | ^"stddev" | ^"standard_deviation" |
^"percentile" | ^"p" | ^"pct" | ^"percentiles" |
^"percentile_rank" | ^"percentile_ranks" | ^"pct_rank" | ^"pct_ranks" |
^"distinct" | ^"unique" | ^"values"
}
// Aggregation modifiers (top N, bottom N)
agg_modifier = { (^"top" | ^"bottom") ~ integer }
// Percentile values
percentile_values = { number ~ ("," ~ number)* }
// Aggregation with field
agg_field = { field_with_mutators | "*" }
// Full aggregation specification
// Percentile values can be inside parens: percentile(field, 50, 75) or outside: percentile(field) 50, 75
aggregation = {
agg_func_name ~ "(" ~ agg_field? ~ ("," ~ percentile_values)? ~ ")" ~
(agg_modifier)? ~
(percentile_values)? ~
(^"as" ~ identifier)?
}
// Group by fields
group_by_field = { field_with_mutators ~ (^"top" ~ integer)? }
group_by_list = { group_by_field ~ ("," ~ group_by_field)* }
// Visualization hint with optional parameters
viz_value = { string | number | boolean | identifier }
viz_param = { identifier ~ "=" ~ viz_value }
viz_params = { "(" ~ viz_param ~ ("," ~ viz_param)* ~ ")" }
viz_hint = { "=>" ~ identifier ~ viz_params? }
// Stats expression (pipe prefix is optional for standalone stats)
stats_expr = {
"|"? ~ ^"stats" ~
aggregation ~ ("," ~ aggregation)* ~
(^"by" ~ group_by_list)? ~
viz_hint?
}
// ========== Top-Level Query ==========
// Combined query with stats
// The mutator rule now prevents consuming | stats, so no lookahead needed
query_with_stats = { logical_expr ~ stats_expr }
// Full query
query = {
SOI ~
(query_with_stats | stats_expr | logical_expr) ~
EOI
}