// Full Trilogy grammar — alternative tokenizer to the lark grammar at
// trilogy/parsing/trilogy.lark. Rule names mirror the lark grammar 1:1
// so the existing NODE_KIND / TOKEN_KIND maps in trilogy/parsing/v2/syntax.py
// apply without modification. Uppercase rule names emit SyntaxTokens;
// lowercase emit SyntaxNodes.
// =============================================================================
// WHITESPACE + COMMENTS
// =============================================================================
// Comments are silently skipped everywhere. Lark's v2 grammar captures
// PARSE_COMMENT only at top-level and end-of-block; we drop them — the v2
// hydration treats comments as optional metadata.
WHITESPACE = _{ " " | "\t" | "\r" | "\n" | "\u{0c}" | "\u{0b}" }
COMMENT = _{
("#" ~ (!("\n" | EOI) ~ ANY)* ~ ("\n" | EOI))
| ("//" ~ (!("\n" | EOI) ~ ANY)* ~ ("\n" | EOI))
| ("/*" ~ (!"*/" ~ ANY)* ~ "*/")
}
// =============================================================================
// ENTRY
// =============================================================================
start = { SOI ~ (block | show_statement)* ~ EOI }
block = { statement ~ _TERMINATOR }
_TERMINATOR = _{ ";" }
statement = _{
function
| type_declaration
| multi_select_statement
| tvf_select_statement
| natural_select_statement
| select_statement
| persist_statement
| rowset_derivation_statement
| selective_import_statement
| self_import_statement
| import_statement
| copy_statement
| merge_statement
| rawsql_statement
| validate_statement
| mock_statement
| publish_statement
| create_statement
| chart_statement
| datasource
| concept
}
// =============================================================================
// CONCEPT DECLARATIONS
// =============================================================================
concept = {
parameter_declaration
| concept_property_declaration
| concept_derivation
| concept_declaration
| properties_declaration
}
parameter_default = { ^"default" ~ literal }
parameter_declaration = { (^"parameter" | ^"param") ~ IDENTIFIER ~ (validated_type | data_type) ~ concept_nullable_modifier? ~ parameter_default? ~ metadata? }
concept_declaration = { select_hide_modifier? ~ PURPOSE ~ IDENTIFIER ~ (validated_type | data_type) ~ concept_nullable_modifier? ~ metadata? }
concept_property_declaration = { select_hide_modifier? ~ UNIQUE? ~ PROPERTY ~ (prop_ident | prop_ident_wildcard | IDENTIFIER) ~ (validated_type | data_type) ~ concept_nullable_modifier? ~ metadata? }
// body is `conditional` (not `expr`) so a derived concept can name a full
// boolean predicate (`and`/`or`, `between`, `is null`) — the same grammar
// `?`/`where` accept. `conditional` collapses to a bare `expr` for the
// single-comparison/arithmetic cases, so prior derivations are unaffected.
concept_derivation = { select_hide_modifier? ~ (PURPOSE | AUTO | PROPERTY) ~ (prop_ident | prop_ident_wildcard | IDENTIFIER) ~ "<-" ~ conditional }
concept_nullable_modifier = { "?" }
inline_property = { select_hide_modifier? ~ IDENTIFIER ~ (validated_type | data_type) ~ concept_nullable_modifier? ~ metadata? }
inline_property_list = { inline_property ~ ("," ~ inline_property)* ~ ","? }
properties_declaration = { ^"properties" ~ (prop_ident_list | IDENTIFIER) ~ "(" ~ inline_property_list ~ ")" }
prop_ident_list = { "<" ~ IDENTIFIER ~ ("," ~ IDENTIFIER)* ~ ","? ~ ">" }
prop_ident = { "<" ~ IDENTIFIER ~ ("," ~ IDENTIFIER)* ~ ","? ~ ">" ~ "." ~ IDENTIFIER }
prop_ident_wildcard = { "<" ~ "*" ~ ">" ~ "." ~ IDENTIFIER }
// =============================================================================
// DATASOURCE
// =============================================================================
datasource = {
DATASOURCE_ROOT? ~ (DATASOURCE_PARTIAL | SHORTHAND_MODIFIER)? ~ ^"datasource" ~ IDENTIFIER
~ "(" ~ column_assignment_list ~ ")"
~ grain_clause?
~ whole_grain_clause?
~ (address | query | file)
~ where?
~ datasource_update_trigger_clause?
~ datasource_lag_clause?
~ datasource_refresh_clause?
~ datasource_partition_clause?
~ datasource_status_clause?
}
DATASOURCE_ROOT = @{ ^"root" }
DATASOURCE_PARTIAL = @{ ^"partial" }
whole_grain_clause = { ^"complete" ~ where }
grain_clause = { ^"grain" ~ "(" ~ column_list ~ ")" }
address = { ^"address" ~ (F_QUOTED_ADDRESS | QUOTED_ADDRESS | ADDRESS) }
query = { ^"query" ~ MULTILINE_STRING }
file_path_list = { "[" ~ (F_FILE_PATH | FILE_PATH) ~ ("," ~ (F_FILE_PATH | FILE_PATH))* ~ ","? ~ "]" }
file_const_ref = { IDENTIFIER }
file = { ^"file" ~ ((F_FILE_PATH ~ ":" ~ F_FILE_PATH) | (FILE_PATH ~ ":" ~ FILE_PATH) | F_FILE_PATH | FILE_PATH | file_path_list | file_const_ref) }
STATE = @{ ^"state" }
datasource_status_clause = { STATE ~ DATASOURCE_STATUS }
DATASOURCE_STATUS = @{ ^"published" | ^"unpublished" }
DATASOURCE_UPDATE_TRIGGER = @{ ^"incremental" | ^"freshness" }
DURATION_UNIT = @{ (^"second" | ^"minute" | ^"hour" | ^"day" | ^"week" | ^"month" | ^"quarter" | ^"year") ~ ^"s"? }
datasource_lag_clause = { ^"within" ~ int_lit ~ DURATION_UNIT? }
datasource_update_trigger_clause = { DATASOURCE_UPDATE_TRIGGER ~ ^"by" ~ (column_list | FILE_PATH) }
datasource_refresh_clause = { ^"refresh" ~ FILE_PATH }
datasource_partition_clause = { ^"partition" ~ ^"by" ~ column_list }
concept_assignment = { SHORTHAND_MODIFIER* ~ IDENTIFIER }
column_assignment = {
(raw_column_assignment ~ ":" ~ concept_assignment)
| (QUOTED_IDENTIFIER ~ ":" ~ concept_assignment)
| (IDENTIFIER ~ ":" ~ concept_assignment)
| (expr ~ ":" ~ concept_assignment)
| concept_assignment
}
RAW_ENTRY = @{ ^"raw" ~ WHITESPACE* ~ "(" }
raw_column_assignment = { RAW_ENTRY ~ MULTILINE_STRING ~ ")" }
column_assignment_list = { column_assignment ~ ("," ~ column_assignment)* ~ ","? }
column_list = { IDENTIFIER ~ ("," ~ IDENTIFIER)* ~ ","? }
// =============================================================================
// IMPORT STATEMENTS
// =============================================================================
import_statement = { ^"import" ~ IMPORT_DOT* ~ dotted_identifier_tail ~ (^"as" ~ IDENTIFIER)? }
selective_import_statement = { ^"from" ~ IMPORT_DOT* ~ dotted_identifier_tail ~ (^"as" ~ IDENTIFIER)? ~ ^"import" ~ import_concepts }
self_import_statement = { SELF_IMPORT ~ ^"import" ~ ^"as" ~ IDENTIFIER }
import_concepts = { IDENTIFIER ~ ("," ~ IDENTIFIER)* ~ ","? }
dotted_identifier_tail = _{ IDENTIFIER ~ ("." ~ IDENTIFIER)* }
IMPORT_DOT = @{ "." }
SELF_IMPORT = @{ ^"self" }
// =============================================================================
// PERSIST
// =============================================================================
persist_statement = { full_persist | auto_persist }
PERSIST_MODE = @{ ^"append" | ^"overwrite" | ^"persist" }
persist_partition_clause = { ^"by" ~ column_list }
auto_persist = { PERSIST_MODE ~ IDENTIFIER ~ where? }
full_persist = {
PERSIST_MODE
~ (!^"into" ~ IDENTIFIER)?
~ ^"into" ~ IDENTIFIER
~ persist_partition_clause?
~ ^"from" ~ select_statement
}
// =============================================================================
// SELECT
// =============================================================================
from_clause = { ^"from" ~ IDENTIFIER ~ ("," ~ IDENTIFIER)* ~ ","? }
// join_clause may appear before "select" (legacy) OR right after the select
// list (preferred, SQL-like); both positions collect into the same join set.
select_statement = {
from_clause? ~ where? ~ join_clause* ~ ^"select" ~ select_list ~ join_clause* ~ where? ~ select_grouping? ~ having? ~ order_by? ~ limit?
}
// query-scoped join: SUBSET/UNION are the only spellings (left/full/inner/right/
// cross still lex but are rejected at hydration with a migration hint).
// blends two otherwise-disjoint concepts on a shared key for
// this select only (a query-local merge). LEFT anchors its first key (others
// partial); FULL coalesces. SUBSET/UNION are the domain-declaration spellings
// (subset join a = b: a's values ⊆ b's, b anchors; union join: neither side
// contains the other) — normalized onto the same two relations at hydration.
// `a = b = c` chains all keys into ONE equivalence group with this join type.
// `a = b and c = d` is sugar for two separate join clauses of this type (two
// DISTINCT equivalence groups), saving a repeated `JOIN_TYPE join` prefix.
join_clause = { JOIN_TYPE ~ ^"join" ~ join_group ~ (^"and" ~ join_group)* }
// Each key is a `sum_operator` (the expr level just below comparison) so join
// keys admit arbitrary expressions (arithmetic, aggregates, windows, ...), not
// just bare concept references; a comparison key can be parenthesized. The
// separator reuses COMPARISON_OPERATOR to mirror lark (where a bare `"="` would
// mint a terminal that breaks comparisons); the hydrator enforces `=`.
join_group = { sum_operator ~ (COMPARISON_OPERATOR ~ sum_operator)+ }
multi_select_statement = {
select_statement
~ (^"merge" ~ select_statement)+
~ ^"align" ~ align_clause
~ (^"derive" ~ derive_clause)?
~ select_grouping? ~ having? ~ order_by? ~ limit?
}
align_item = { select_hide_modifier? ~ IDENTIFIER ~ ":" ~ IDENTIFIER ~ ("," ~ IDENTIFIER)* ~ ","? }
align_clause = { align_item ~ (^"and" ~ align_item)* ~ (^"and")? }
derive_item = { expr ~ ("->" | ^"as") ~ IDENTIFIER }
derive_clause = { derive_item ~ ("," ~ derive_item)* ~ ","? }
merge_statement = { ^"merge" ~ WILDCARD_IDENTIFIER ~ ^"into" ~ SHORTHAND_MODIFIER? ~ WILDCARD_IDENTIFIER }
rawsql_statement = { ^"raw_sql" ~ "(" ~ MULTILINE_STRING ~ ")" }
VALIDATE_SCOPE = @{ ^"concepts" | ^"datasources" | ^"datasource" | ^"concept" }
validate_statement = {
(^"validate" ~ ^"all")
| (^"validate" ~ VALIDATE_SCOPE ~ (IDENTIFIER ~ ("," ~ IDENTIFIER)* ~ ","?)?)
// the optional label must not eat the `select` keyword: PEG `?` commits once
// matched, so guard it with "what follows is not itself the natural select".
| (^"validate" ~ (!natural_select_statement ~ IDENTIFIER)? ~ natural_select_statement ~ ^"matches" ~ "(" ~ select_statement ~ ")" ~ validate_query_config?)
}
// natural-language select: an agent answers the question with a generated query.
// Must precede select_statement in the statement alternation: PEG commits to a
// successful alternative, and select_statement would consume `select natural`
// as a one-item select list and strand the question string.
natural_select_statement = { ^"select" ~ ^"natural" ~ string_lit }
validate_query_option = { IDENTIFIER ~ "=" ~ (literal | IDENTIFIER) }
validate_query_config = { ^"with" ~ "(" ~ validate_query_option ~ ("," ~ validate_query_option)* ~ ","? ~ ")" }
mock_statement = { ^"mock" ~ VALIDATE_SCOPE ~ (IDENTIFIER ~ ("," ~ IDENTIFIER)* ~ ","?)? }
PUBLISH_ACTION = @{ ^"publish" | ^"unpublish" }
publish_statement = { PUBLISH_ACTION ~ VALIDATE_SCOPE ~ IDENTIFIER }
create_modifier_clause = { CREATE_IF_NOT_EXISTS | CREATE_OR_REPLACE }
CREATE_IF_NOT_EXISTS = @{ ^"if" ~ WHITESPACE+ ~ ^"not" ~ WHITESPACE+ ~ ^"exists" }
CREATE_OR_REPLACE = @{ ^"or" ~ WHITESPACE+ ~ ^"replace" }
CREATE_WITH_DATA = @{ ^"with" ~ WHITESPACE+ ~ ^"data" }
create_statement = { ^"create" ~ create_modifier_clause? ~ VALIDATE_SCOPE ~ IDENTIFIER ~ CREATE_WITH_DATA? }
COPY_TYPE = @{ ^"csv" | ^"json" | ^"parquet" | ^"png" | ^"svg" | ^"html" | ^"pdf" }
copy_option = { IDENTIFIER ~ "=" ~ literal }
copy_options = { "(" ~ copy_option ~ ("," ~ copy_option)* ~ ","? ~ ")" }
copy_statement = { ^"copy" ~ ^"into" ~ COPY_TYPE ~ (FILE_PATH | string_lit) ~ copy_options? ~ ^"from" ~ (chart_statement | select_statement) }
// `barh` must precede `bar` in PEG ordered choice so pest doesn't consume
// `bar` as a complete match and leave `h` for the next rule.
CHART_TYPE = @{ ^"line" | ^"barh" | ^"bar" | ^"point" | ^"area" | ^"headline" | ^"donut" | ^"heatmap" | ^"boxplot" | ^"treemap" }
CHART_PLACE_TYPE = @{ ^"hline" | ^"vline" }
CHART_BOOL_FIELD = @{ ^"hide_legend" | ^"show_title" }
CHART_SCALE_FIELD = @{ ^"scale_x" | ^"scale_y" }
SCALE_TYPE = @{ ^"linear" | ^"log" | ^"sqrt" }
chart_bool_setting = { ^"set" ~ CHART_BOOL_FIELD }
chart_scale_setting = { ^"set" ~ CHART_SCALE_FIELD ~ ":" ~ SCALE_TYPE }
chart_layer_binding = { IDENTIFIER ~ "<-" ~ expr ~ (^"as" ~ IDENTIFIER)? }
chart_layer_body = { chart_layer_binding ~ ("," ~ chart_layer_binding)* ~ ","? }
chart_layer = { ^"layer" ~ CHART_TYPE ~ "(" ~ chart_layer_body ~ ")" ~ (^"from" ~ select_statement)? ~ order_by? ~ limit? }
chart_place = { ^"place" ~ CHART_PLACE_TYPE ~ ^"at" ~ literal ~ (^"as" ~ IDENTIFIER)? }
chart_component = { chart_layer | chart_place | chart_scale_setting | chart_bool_setting }
chart_statement = { ^"chart" ~ chart_component+ }
// =============================================================================
// FUNCTION DEFINITIONS
// =============================================================================
function = { table_function | raw_function }
function_binding_type = { ":" ~ (validated_type | data_type) }
function_binding_default = { "=" ~ expr }
function_binding_item = { IDENTIFIER ~ function_binding_type? ~ function_binding_default? }
function_binding_list = { function_binding_item ~ ("," ~ function_binding_item)* ~ ","? }
_DEF_TABLE = @{ ^"def" ~ (" " | "\t" | "\r" | "\n")+ ~ ^"table" ~ (" " | "\t" | "\r" | "\n")+ }
raw_function = { ^"def" ~ IDENTIFIER ~ ("(" ~ function_binding_list? ~ ")")? ~ "->" ~ conditional }
table_function = { _DEF_TABLE ~ IDENTIFIER ~ "(" ~ function_binding_list? ~ ")" ~ "->" ~ ^"select" ~ expr ~ subselect_where? ~ subselect_order? ~ subselect_limit? }
// =============================================================================
// TYPE DECLARATION
// =============================================================================
type_drop_clause = { ^"drop" ~ IDENTIFIER ~ ("|" ~ IDENTIFIER)* }
type_declaration = { ^"type" ~ IDENTIFIER ~ (validated_type | data_type) ~ ("|" ~ (validated_type | data_type))* ~ type_drop_clause? }
// =============================================================================
// ROWSET
// =============================================================================
rowset_derivation_statement = {
(^"rowset" ~ IDENTIFIER ~ "<-" ~ (_tvf_invocation | multi_select_statement | select_statement))
| (^"with" ~ IDENTIFIER ~ ^"as" ~ (_tvf_invocation | multi_select_statement | select_statement))
}
// =============================================================================
// SHOW STATEMENT
// =============================================================================
show_category = { CONCEPTS | DATASOURCES }
CONCEPTS = @{ ^"concepts" }
DATASOURCES = @{ ^"datasources" }
show_statement = { ^"show" ~ (show_category | validate_statement | natural_select_statement | select_statement | persist_statement) ~ _TERMINATOR }
// =============================================================================
// SELECT-RELATED
// =============================================================================
select_hide_modifier = { "--" }
select_partial_modifier = { "~" }
// The alias tail is optional: `select a, b+1;` materializes an anonymous
// concept with a derived name. A bare concept ref parses as an unaliased
// select_transform whose expr collapses to the concept_lit (a dedicated
// concept_lit alternative would either be dead — expr matches a bare ref
// first — or commit early and break `a + 1`; mirrors trilogy.lark).
select_item = { (select_hide_modifier | select_partial_modifier)? ~ select_transform }
select_list = { select_item ~ ("," ~ !SELECT_LIST_STOP ~ select_item)* ~ ","? }
select_transform = { expr ~ (("->" | ^"as") ~ IDENTIFIER ~ metadata?)? }
metadata = { ^"metadata" ~ "(" ~ IDENTIFIER ~ "=" ~ string_lit ~ ")" }
limit = { ^"limit" ~ INT_DIGITS }
INT_DIGITS = @{ ASCII_DIGIT+ }
order_by = { ^"order" ~ ^"by" ~ order_list }
order_list = { _order_atom ~ ("," ~ _order_atom)* ~ ","? }
ORDER_IDENTIFIER = @{ (ASCII_ALPHA | "_") ~ (ASCII_ALPHANUMERIC | "_" | ".")* ~ (" " | "\t")+ }
// Atomic so the word-boundary `!(ALNUM|_)` is checked immediately after the
// keyword — otherwise pest skips intervening whitespace between the keyword
// and the boundary check, letting `desc\n limit` match and causing the
// `expr ~ ordering` fallback to greedily consume the identifier as a window
// function.
ORDER_DIRECTION_LA = @{ (^"asc" | ^"desc" | ^"nulls") ~ !(ASCII_ALPHANUMERIC | "_") }
_order_atom = _{
(ORDER_IDENTIFIER ~ &ORDER_DIRECTION_LA ~ ordering)
| (expr ~ ordering)
}
ORDERING_DIRECTION = @{ ^"asc" | ^"desc" }
NULLS_SORT = @{ ^"first" | ^"last" | ^"auto" }
ordering = { ORDERING_DIRECTION? ~ (^"nulls" ~ NULLS_SORT)? }
// =============================================================================
// WHERE / HAVING / CONDITIONAL
// =============================================================================
where = { ^"where" ~ conditional }
having = { ^"having" ~ conditional }
// The v2 hydration `conditional` uses right-associative munching:
// Conditional(left=args[0], operator=args[1], right=munch(args[2:])).
// We emit nested conditional nodes so AND/OR precedence is baked into the tree:
// a AND b OR c -> conditional(conditional(a, AND, b), OR, c)
// a OR b AND c -> conditional(a, OR, conditional(b, AND, c))
conditional = { _and_conditional ~ (LOGICAL_OR ~ _and_conditional)* }
_and_conditional = { _condition_unit ~ (LOGICAL_AND ~ _condition_unit)* }
// `(a and b)` can only ever match condition_parenthetical — `expr` has no
// AND/OR — but trying `expr` first means every parenthesized boolean group
// is parsed as `expr` (and discarded) 3-4x before the fallback fires. Try
// the bare paren form first, guarded by `!_EXPR_CONTINUATION` so it does not
// steal `(x) = 0`, `(x) + 1`, `(x).attr` etc., which are genuine `expr`s.
// The `not (...)` form stays after `expr` so `not (single_expr)` keeps
// resolving to `fnot` rather than a paren-condition.
_condition_unit = _{ condition_parenthetical | expr | condition_parenthetical_not }
condition_parenthetical = { "(" ~ conditional ~ ")" ~ !_EXPR_CONTINUATION }
condition_parenthetical_not = { CONDITION_NOT ~ "(" ~ conditional ~ ")" }
// A `)` followed by any of these is the tail of an `expr`, not the end of a
// parenthesized condition — defer to the `expr` alternative when seen.
// BETWEEN_LA is atomic so the keyword-boundary check is applied immediately
// after the word (a non-atomic sequence would skip whitespace before it).
// `?` is the filter operator: `(x + y) ? cond` is a filter_item whose head is a
// parenthesized expr, not a parenthesized condition. Defer to `expr` so the
// filter parses (matters now that a derived-concept body is a `conditional`).
_EXPR_CONTINUATION = _{
COMPARISON_OPERATOR | PLUS_OR_MINUS | MULTIPLY_DIVIDE_PERCENT
| "." | "[" | "::" | BETWEEN_LA | "?"
}
BETWEEN_LA = @{ ^"between" ~ !(ASCII_ALPHANUMERIC | "_") }
// Keyword boundary guard: not followed by identifier chars. Prevents `or`
// from matching the "or" prefix of `order`, `and` from matching `android`, etc.
KW_END = _{ !(ASCII_ALPHANUMERIC | "_") }
LOGICAL_OR = @{ ^"or" ~ KW_END }
LOGICAL_AND = @{ ^"and" ~ KW_END }
CONDITION_NOT = @{ ^"not" ~ KW_END }
// =============================================================================
// EXPRESSIONS
// =============================================================================
// Lark: ?expr: comparison_root | between_root
expr = _{ between_comparison | comparison }
// ?comparison_root: sum_chain (COMPARISON_OPERATOR sum_chain)? -> comparison
comparison = { sum_operator ~ (COMPARISON_OPERATOR ~ sum_operator)? }
// ?between_root: sum_chain "between" sum_chain "and" sum_chain
between_comparison = { sum_operator ~ ^"between" ~ sum_operator ~ ^"and" ~ sum_operator }
// ?sum_chain: UNARY_MINUS? product_chain (PLUS_OR_MINUS product_chain)*
// A leading UNARY_MINUS is prefix negation on the first product_chain
// (`-sum(x)`, `-col`, `-(a+b)`); it binds only the first operand so
// `-a + b` is `(-a) + b`. UNARY_MINUS's negative lookahead excludes a `-`
// glued to a number, so a negative literal (`-1`) stays a single literal.
sum_operator = { UNARY_MINUS? ~ product_operator ~ (PLUS_OR_MINUS ~ product_operator)* }
// `-` not glued to a number (`-sum`, `- 1`, `-(x)`) and not the `->` arrow.
UNARY_MINUS = @{ "-" ~ !(ASCII_DIGIT | "." | ">") }
// ?product_chain: atom (MULTIPLY_DIVIDE_PERCENT atom)*
product_operator = { access_chain ~ (MULTIPLY_DIVIDE_PERCENT ~ access_chain)* }
// Access expressions are a post-atom wrapping. Parsed as a single chain so
// _atom is parsed exactly once per position (avoids exponential backtracking).
// The adapter rewrites the access_chain node name based on the suffix shape:
// no tails → unwrap to atom
// 1 dot_tail → attr_access
// 1 bracket_tail with int_lit → index_access
// 1 bracket_tail with string_lit → map_key_access
// 1 dcolon_tail → fcast
// multiple tails → chained_access (with a trailing fcast wrapping if needed)
access_chain = { _atom ~ _access_tail* }
_access_tail = _{ dot_tail | bracket_tail | dcolon_tail }
dot_tail = { "." ~ (string_lit | ATTR_NAME) }
bracket_tail = { "[" ~ (int_lit | string_lit) ~ "]" }
// Allow a bare IDENTIFIER so users can cast straight to a registered trait
// (e.g. `x::percent` rather than `x::float::percent`). data_type is tried
// first so `float::percent` keeps the composed-trait shape.
dcolon_tail = { "::" ~ (data_type | IDENTIFIER) }
// getattr(atom, "name") — still available as a distinct atom-level form.
attr_access_paren = { ^"getattr" ~ "(" ~ expr ~ "," ~ string_lit ~ ")" }
// _atom covers lark's ?atom alternatives (minus access). Order-sensitive in PEG.
// filter_item must be first so the `X ? cond` short form is tried before
// alternatives like literal/parenthetical that would otherwise consume X alone.
_atom = _{
filter_item
| scalar_subquery
| parenthetical
| expr_tuple
| literal
| attr_access_paren
| custom_function
| _constant_functions
| _static_functions
| _generic_functions
| _date_functions
| aggregate_by
| window_item
| aggregate_functions
| unnest
| union
| subselect
| fgroup
| concept_lit
}
// attr_access rule name matches lark; we rename attr_access_dot and attr_access_paren
// in the adapter to "attr_access".
ATTR_NAME = @{ (ASCII_ALPHA | "_") ~ (ASCII_ALPHANUMERIC | "_")* }
parenthetical = { "(" ~ expr ~ ")" }
expr_tuple = { "(" ~ expr ~ ("," ~ expr)+ ~ ","? ~ ")" | "(" ~ expr ~ "," ~ ")" }
// An inline `(select …)` scalar/membership subquery. Desugared at hydration
// into an anonymous rowset whose single output is referenced here. Ordered
// before `parenthetical` in _atom: `expr` can never start with select/where/
// from, so a non-select paren cleanly backtracks to `parenthetical`.
scalar_subquery = { "(" ~ select_statement ~ ")" }
COMPARISON_OPERATOR = @{
(^"is" ~ WHITESPACE+ ~ ^"not" ~ WHITESPACE+)
| (^"is" ~ WHITESPACE+)
| (^"not" ~ WHITESPACE+ ~ ^"in" ~ WHITESPACE+)
| (^"in" ~ WHITESPACE+)
| "="
| ">="
| "<="
| "!="
| ">"
| "<"
}
PLUS_OR_MINUS = @{ "+" | ("-" ~ !">") | "||" | (^"not" ~ WHITESPACE+ ~ ^"ilike") | (^"not" ~ WHITESPACE+ ~ ^"like") | ^"ilike" | ^"like" }
MULTIPLY_DIVIDE_PERCENT = @{ "**" | "*" | "/" | "%" }
_UNNEST = _{ ^"unnest(" }
unnest = { _UNNEST ~ expr ~ ")" }
_SUBSELECT = _{ ^"subselect(" }
subselect_where = { ^"where" ~ conditional }
subselect_order = { ^"order" ~ ^"by" ~ order_list }
subselect_limit = { ^"limit" ~ INT_DIGITS }
subselect = { _SUBSELECT ~ expr ~ subselect_where? ~ subselect_order? ~ subselect_limit? ~ ")" }
_UNION = _{ ^"union(" }
union = { _UNION ~ (expr ~ ",")* ~ expr ~ ")" }
// table-valued functions: relational union(...)/except(...)/intersect(...).
// union reuses _UNION at statement positions (rowset-RHS / inline-FROM),
// disjoint from the atom-level `union`; except/intersect exist only at these
// statement positions. except/intersect are SQL set operators (deduplicated,
// NULL-safe row comparison); except left-folds over 3+ arms.
_EXCEPT_TVF = _{ ^"except(" }
_INTERSECT_TVF = _{ ^"intersect(" }
tvf_output_item = { select_hide_modifier? ~ IDENTIFIER ~ (PURPOSE? ~ (validated_type | data_type) ~ concept_nullable_modifier?)? ~ metadata? }
tvf_output = { "->" ~ "(" ~ tvf_output_item ~ ("," ~ tvf_output_item)* ~ ","? ~ ")" }
tvf_rel_arg = { "(" ~ select_statement ~ ")" }
_tvf_arg = _{ tvf_rel_arg }
tvf_union_invocation = { _UNION ~ (_tvf_arg ~ ("," ~ _tvf_arg)* ~ ","?)? ~ ")" ~ tvf_output }
tvf_except_invocation = { _EXCEPT_TVF ~ (_tvf_arg ~ ("," ~ _tvf_arg)* ~ ","?)? ~ ")" ~ tvf_output }
tvf_intersect_invocation = { _INTERSECT_TVF ~ (_tvf_arg ~ ("," ~ _tvf_arg)* ~ ","?)? ~ ")" ~ tvf_output }
_tvf_invocation = _{ tvf_union_invocation | tvf_except_invocation | tvf_intersect_invocation }
tvf_select_statement = { ^"from" ~ _tvf_invocation ~ select_statement }
// =============================================================================
// AGGREGATES
// =============================================================================
aggregate_all = { "*" }
grouping_set = { "(" ~ expr_over_list? ~ ")" }
// Paren-wrapped expression list: `by (substring(x,1,2), other)`. Parens supply
// the unambiguous boundary needed for arbitrary expressions; the build phase
// materializes non-concept entries via instantiate_concept.
aggregate_paren_by = { "(" ~ expr_over_list ~ ")" }
// An aggregate's `by <grain>` never starts with a grouping keyword — those are
// reserved for the SELECT-level `by rollup (…)` clause. The lookahead refuses
// `by rollup`/`by cube`/`by grouping sets` here so they fall through to it.
GROUPING_LEAD = @{ (^"rollup" | ^"cube" | ^"grouping") ~ !(ASCII_ALPHANUMERIC | "_" | ".") }
aggregate_over = { ^"by" ~ !GROUPING_LEAD ~ (aggregate_all | aggregate_paren_by | over_list) }
// SELECT-level multi-level grouping (ROLLUP / CUBE / GROUPING SETS): a property
// of the whole select, propagated by the planner to every aggregate with no
// explicit `by` grain so all measures share ONE grouping pass.
select_rollup = { ^"by" ~ ^"rollup" ~ "(" ~ expr_over_list? ~ ")" }
select_cube = { ^"by" ~ ^"cube" ~ "(" ~ expr_over_list ~ ")" }
select_grouping_sets = { ^"by" ~ ^"grouping" ~ ^"sets" ~ "(" ~ grouping_set ~ ("," ~ grouping_set)* ~ ")" }
select_grouping = { select_rollup | select_cube | select_grouping_sets }
// Mirrors lark's single-token regex: /(group)\s+([a-zA-Z_][a-zA-Z0-9_.]*)/i
// Emits "group <identname>" as a single string so the hydrator's
// `str(args[0]).split(" ")[-1]` works unchanged.
GROUP_TARGET = @{ ^"group" ~ (" " | "\t" | "\r" | "\n")+ ~ (ASCII_ALPHA | "_") ~ (ASCII_ALPHANUMERIC | "_" | ".")* }
aggregate_by = { GROUP_TARGET ~ ^"by" ~ IDENTIFIER ~ ("," ~ IDENTIFIER)* }
count = { ^"count" ~ "(" ~ expr ~ ")" }
// Accept the SQL-habit `count(distinct x)` as an alias for `count_distinct(x)`.
// Tried before `count` in aggregate_functions, so `count(x)` still resolves to count.
count_distinct = { (^"count_distinct" ~ "(" ~ expr ~ ")") | (^"count" ~ "(" ~ ^"distinct" ~ expr ~ ")") }
grouping = { ^"grouping" ~ "(" ~ expr ~ ")" }
grouping_id = { ^"grouping_id" ~ "(" ~ (expr ~ ",")* ~ expr ~ ")" }
sum = { ^"sum" ~ "(" ~ expr ~ ")" }
avg = { ^"avg" ~ "(" ~ expr ~ ")" }
stddev = { ^"stddev" ~ "(" ~ expr ~ ")" }
variance = { ^"variance" ~ "(" ~ expr ~ ")" }
max = { ^"max" ~ "(" ~ expr ~ ")" }
min = { ^"min" ~ "(" ~ expr ~ ")" }
array_agg = { ^"array_agg" ~ "(" ~ expr ~ ")" }
bool_and = { ^"bool_and" ~ "(" ~ expr ~ ")" }
bool_or = { ^"bool_or" ~ "(" ~ expr ~ ")" }
any = { ^"any" ~ "(" ~ expr ~ ")" }
aggregate_functions = {
(count_distinct | count | grouping_id | grouping | sum | avg | stddev | variance | max | min | array_agg | bool_and | bool_or | any) ~ aggregate_over?
}
fgroup = { ^"group" ~ "(" ~ expr ~ ")" ~ aggregate_over? }
// =============================================================================
// FILTER
// =============================================================================
// PEG ordered-choice: longer prefixes first. `IDENTIFIER` would otherwise
// match `upper` alone, commit, and fail at `?` before `_static_functions`
// gets a chance to match `upper(...)`. Mirrors the lark `_filter_alt` LHS:
// every non-left-recursive atom alternative, plus parenthesized `expr` for
// arbitrary sub-trees.
// The `filter` keyword needs a word-boundary guard: without it, an unaliased
// select item like `filtered_even_orders` reads as `filter` + `ed_even_orders`
// (pest is scannerless and inter-element whitespace is optional). FILTER_KW is
// used only inside `&` so it emits no pair and both backends' trees stay
// identical; the "." exclusion keeps a `filter.x` concept reference out of the
// keyword path too. (Mirrors the WINDOW_LEGACY_KW lookahead pattern.)
FILTER_KW = @{ ^"filter" ~ !(ASCII_ALPHANUMERIC | "_" | ".") }
filter_item = {
(&FILTER_KW ~ ^"filter" ~ IDENTIFIER ~ where)
| ((_constant_functions | _static_functions | _generic_functions | _date_functions | window_item | aggregate_functions | aggregate_by | custom_function | unnest | union | subselect | fgroup | expr_tuple | literal | ("(" ~ expr ~ ")") | IDENTIFIER) ~ "?" ~ conditional)
}
// =============================================================================
// WINDOW FUNCTIONS
// =============================================================================
WINDOW_TYPE_LEGACY = @{
(^"row_number" | ^"rank" | ^"dense_rank" | ^"lag" | ^"lead" | ^"sum" | ^"avg" | ^"max" | ^"min" | ^"count")
~ (" " | "\t")+
}
WINDOW_TYPE_SQL_NUMBERING = @{
(^"row_number" | ^"rank" | ^"dense_rank")
~ (" " | "\t")*
~ "("
}
WINDOW_TYPE_SQL_NAVIGATION = @{
(^"lag" | ^"lead")
~ (" " | "\t")*
~ "("
}
window_item_over = { ^"over" ~ over_list }
window_item_order = { ^"order"? ~ ^"by" ~ order_list }
// PARTITION BY accepts arbitrary expressions (e.g. `grouping(a)+grouping(b)`);
// see the mirroring comment in trilogy.lark for the LALR-conflict rationale.
window_sql_partition = { ^"partition" ~ ^"by" ~ expr_over_list }
window_sql_order = { ^"order" ~ ^"by" ~ order_list }
_WINDOW_OVER_PAREN = _{ ^"over" ~ (" " | "\t")* ~ "(" }
window_sql_over = { _WINDOW_OVER_PAREN ~ window_sql_partition? ~ window_sql_order? ~ ")" }
// Two-alternative form: PEG can't use LALR lookahead to decide whether
// `int_lit` consumes an offset or belongs to the expr body. The explicit
// int_lit path is tried first so `rank 2 x` stays an offset; `rank 1 by
// x desc` falls back to the plain form with `1` as the expr literal.
// The negative lookahead prevents alt 1 from consuming `1` as the offset
// AND `by` as the expr (concept_lit greedily matches `by` as an identifier
// because `by` can't be reserved — concepts like `id.by` use it as a name).
// `as` is excluded on both alternatives so a concept that shares a legacy
// window name still aliases: `select count as n` is IDENTIFIER + alias, not
// a window over a concept named `as`.
WINDOW_LEGACY_KW = @{ (^"by" | ^"over" | ^"order" | ^"as") ~ !(ASCII_ALPHANUMERIC | "_" | ".") }
window_item_legacy = {
WINDOW_TYPE_LEGACY ~ int_lit ~ !WINDOW_LEGACY_KW ~ expr ~ window_item_over? ~ window_item_order?
| WINDOW_TYPE_LEGACY ~ !WINDOW_LEGACY_KW ~ expr ~ window_item_over? ~ window_item_order?
}
// SQL-like numbering: rank(a, b, c) over (...). All comma-separated fields
// are equal-status grain keys. Empty args inherit the surrounding SELECT grain.
window_item_sql_numbering = { WINDOW_TYPE_SQL_NUMBERING ~ (expr ~ ("," ~ expr)*)? ~ ")" ~ window_sql_over? }
// SQL-like navigation: lag(field) or lag(field, 2) over (...).
window_item_sql_navigation = { WINDOW_TYPE_SQL_NAVIGATION ~ expr ~ ("," ~ int_lit)? ~ ")" ~ window_sql_over? }
// SQL-like aggregate-as-window: sum(x) over (partition by y order by z).
// Match before plain aggregate_functions in the expr atom list so the
// trailing over clause binds here instead of leaving the parser stuck
// after the aggregate call.
window_item_sql_aggregate = { aggregate_functions ~ window_sql_over }
window_item = { window_item_sql_aggregate | window_item_sql_numbering | window_item_sql_navigation | window_item_legacy }
// Allow newlines/CR between the comma and the identifier so multi-line
// `over a, b,\n c` and `by a, b,\n c` lists parse the same as the lark
// grammar (which uses `\s*`). This rule is atomic (`@`) so we have to
// enumerate every whitespace char ourselves — non-atomic implicit
// WHITESPACE doesn't apply.
OVER_COMPONENT_REF = @{ "," ~ (" " | "\t" | "\r" | "\n")* ~ (ASCII_ALPHA | "_") ~ (ASCII_ALPHANUMERIC | "_" | ".")* }
over_component = { OVER_COMPONENT_REF ~ (^"end")? }
over_list = { concept_lit ~ over_component* }
// Paren-bounded over list: arbitrary expressions, comma-separated. Used in
// PARTITION BY (inside OVER paren) and GROUPING SETS (...). Mirrors lark.
expr_over_list = { expr ~ ("," ~ expr)* }
// =============================================================================
// CAST
// =============================================================================
// Only the cast(expr as type) form here — the postfix `atom::type` form lives
// in the access chain as `fcast_postfix` to avoid PEG left-recursion.
// `cast(expr as <type>)` — mirrors dcolon_tail in allowing a bare IDENTIFIER
// so a registered trait (e.g. `percent`) can be used directly as the target.
fcast = { ^"cast" ~ "(" ~ expr ~ ^"as" ~ (data_type | IDENTIFIER) ~ ")" }
// =============================================================================
// GENERIC FUNCTIONS
// =============================================================================
concat = { ^"concat" ~ "(" ~ expr ~ ("," ~ expr)* ~ ")" }
concat_ws = { ^"concat_ws" ~ "(" ~ expr ~ "," ~ expr ~ ("," ~ expr)* ~ ")" }
// one or more keys
fgrain = { ^"grain" ~ "(" ~ expr ~ ("," ~ expr)* ~ ")" }
fcoalesce = { ^"coalesce" ~ "(" ~ expr ~ ("," ~ expr)* ~ ")" }
fgreatest = { ^"greatest" ~ "(" ~ expr ~ ("," ~ expr)* ~ ")" }
fleast = { ^"least" ~ "(" ~ expr ~ ("," ~ expr)* ~ ")" }
fcase_when = { ^"when" ~ conditional ~ ^"then" ~ expr }
fcase_else = { ^"else" ~ expr }
fcase_simple_when = { ^"when" ~ expr ~ ^"then" ~ expr }
fcase_simple = { ^"case" ~ expr ~ fcase_simple_when+ ~ fcase_else? ~ ^"end" }
fcase = { ^"case" ~ fcase_when* ~ fcase_else? ~ ^"end" }
len = { ^"len" ~ "(" ~ expr ~ ")" }
// The &CONDITION_NOT lookahead keeps identifiers with a `not` prefix (`note`,
// `notice`) from being read as a negation of their tail. The boundary check
// must live in an atomic rule: written inline (`^"not" ~ KW_END`), implicit
// whitespace runs before KW_END and `not value` fails to parse.
fnot = { &CONDITION_NOT ~ ^"not" ~ expr }
fbool = { ^"bool" ~ "(" ~ expr ~ ")" }
fnullif = { ^"nullif" ~ "(" ~ expr ~ "," ~ expr ~ ")" }
frecurse_edge = { ^"recurse_edge" ~ "(" ~ expr ~ "," ~ expr ~ ")" }
_generic_functions = _{
fcast | fcase_simple | fcase | concat_ws | concat | fgrain | fcoalesce | fgreatest | fleast | fnullif | len | fnot | fbool | frecurse_edge
}
// =============================================================================
// CONSTANT FUNCTIONS
// =============================================================================
CURRENT_DATE = @{ ^"current_date()" }
CURRENT_DATETIME = @{ ^"current_datetime()" }
CURRENT_TIMESTAMP = @{ ^"current_timestamp()" }
fcurrent_date = { CURRENT_DATE }
fcurrent_datetime = { CURRENT_DATETIME }
fcurrent_timestamp = { CURRENT_TIMESTAMP }
_constant_functions = _{ fcurrent_date | fcurrent_datetime | fcurrent_timestamp }
// =============================================================================
// STRING / MATH / ARRAY / MAP / GEO FUNCTIONS
// =============================================================================
like = { ^"like" ~ "(" ~ expr ~ "," ~ string_lit ~ ")" }
ilike = { ^"ilike" ~ "(" ~ expr ~ "," ~ string_lit ~ ")" }
upper = { ^"upper" ~ "(" ~ expr ~ ")" }
flower = { ^"lower" ~ "(" ~ expr ~ ")" }
fsplit = { ^"split" ~ "(" ~ expr ~ "," ~ string_lit ~ ")" }
fstrpos = { ^"strpos" ~ "(" ~ expr ~ "," ~ expr ~ ")" }
fcontains = { ^"contains" ~ "(" ~ expr ~ "," ~ expr ~ ")" }
ftrim = { ^"trim" ~ "(" ~ expr ~ ")" }
fltrim = { ^"ltrim" ~ "(" ~ expr ~ ")" }
frtrim = { ^"rtrim" ~ "(" ~ expr ~ ")" }
freplace = { ^"replace" ~ "(" ~ expr ~ "," ~ expr ~ "," ~ expr ~ ")" }
fsubstring = { ^"substring" ~ "(" ~ expr ~ "," ~ expr ~ "," ~ expr ~ ")" }
fregexp_extract = { ^"regexp_extract" ~ "(" ~ expr ~ "," ~ expr ~ ("," ~ int_lit)? ~ ")" }
fregexp_contains = { ^"regexp_contains" ~ "(" ~ expr ~ "," ~ expr ~ ")" }
fregexp_replace = { ^"regexp_replace" ~ "(" ~ expr ~ "," ~ expr ~ "," ~ expr ~ ")" }
HASH_TYPE = @{ ^"md5" | ^"sha1" | ^"sha256" | ^"sha512" }
fhash = { ^"hash" ~ "(" ~ expr ~ "," ~ HASH_TYPE ~ ")" }
fhex = { ^"hex" ~ "(" ~ expr ~ ")" }
_string_functions = _{
like | ilike | upper | flower | fsplit | fstrpos | fsubstring | fcontains
| ftrim | fltrim | frtrim | freplace | fregexp_extract | fregexp_contains | fregexp_replace | fhash | fhex
}
fadd = { ^"add" ~ "(" ~ expr ~ "," ~ expr ~ ")" }
fsub = { ^"subtract" ~ "(" ~ expr ~ "," ~ expr ~ ")" }
fmul = { ^"multiply" ~ "(" ~ expr ~ "," ~ expr ~ ")" }
fdiv = { ^"divide" ~ "(" ~ expr ~ "," ~ expr ~ ")" }
fmod = { ^"mod" ~ "(" ~ expr ~ "," ~ (int_lit | concept_lit) ~ ")" }
flog = { ^"log" ~ "(" ~ expr ~ ("," ~ expr)? ~ ")" }
fround = { ^"round" ~ "(" ~ expr ~ ("," ~ expr)? ~ ")" }
ffloor = { ^"floor" ~ "(" ~ expr ~ ")" }
fceil = { ^"ceil" ~ "(" ~ expr ~ ")" }
fabs = { ^"abs" ~ "(" ~ expr ~ ")" }
fsqrt = { ^"sqrt" ~ "(" ~ expr ~ ")" }
frandom = { ^"random" ~ "(" ~ expr ~ ")" }
_math_functions = _{ fmul | fdiv | fadd | fsub | fround | ffloor | fceil | fmod | flog | fabs | fsqrt | frandom }
farray_sum = { ^"array_sum" ~ "(" ~ expr ~ ")" }
farray_distinct = { ^"array_distinct" ~ "(" ~ expr ~ ")" }
farray_to_string = { ^"array_to_string" ~ "(" ~ expr ~ "," ~ expr ~ ")" }
farray_sort = { ^"array_sort" ~ "(" ~ expr ~ ("," ~ ordering)? ~ ")" }
transform_lambda = { "@" ~ IDENTIFIER }
farray_transform = { ^"array_transform" ~ "(" ~ expr ~ "," ~ transform_lambda ~ ")" }
farray_filter = { ^"array_filter" ~ "(" ~ expr ~ "," ~ transform_lambda ~ ")" }
fgenerate_array = { ^"generate_array" ~ "(" ~ expr ~ "," ~ expr ~ "," ~ expr ~ ")" }
_array_functions = _{
farray_sum | farray_distinct | farray_sort | farray_transform | farray_to_string | farray_filter | fgenerate_array
}
fmap_keys = { ^"map_keys" ~ "(" ~ expr ~ ")" }
fmap_values = { ^"map_values" ~ "(" ~ expr ~ ")" }
_map_functions = _{ fmap_keys | fmap_values }
fgeo_from_text = { ^"geo_from_text" ~ "(" ~ expr ~ ")" }
fgeo_point = { ^"geo_point" ~ "(" ~ expr ~ "," ~ expr ~ ")" }
fgeo_distance = { ^"geo_distance" ~ "(" ~ expr ~ "," ~ expr ~ ")" }
fgeo_x = { ^"geo_x" ~ "(" ~ expr ~ ")" }
fgeo_y = { ^"geo_y" ~ "(" ~ expr ~ ")" }
fgeo_centroid = { ^"geo_centroid" ~ "(" ~ expr ~ ")" }
fgeo_transform = { ^"geo_transform" ~ "(" ~ expr ~ "," ~ expr ~ "," ~ expr ~ ")" }
_geo_functions = _{ fgeo_from_text | fgeo_point | fgeo_distance | fgeo_x | fgeo_y | fgeo_centroid | fgeo_transform }
// =============================================================================
// DATE FUNCTIONS
// =============================================================================
fdate = { ^"date" ~ "(" ~ expr ~ ")" }
fdatetime = { ^"datetime" ~ "(" ~ expr ~ ")" }
ftimestamp = { ^"timestamp" ~ "(" ~ expr ~ ")" }
fsecond = { ^"second" ~ "(" ~ expr ~ ")" }
fminute = { ^"minute" ~ "(" ~ expr ~ ")" }
fhour = { ^"hour" ~ "(" ~ expr ~ ")" }
fday = { ^"day" ~ "(" ~ expr ~ ")" }
fday_name = { ^"day_name" ~ "(" ~ expr ~ ")" }
fday_of_week = { ^"day_of_week" ~ "(" ~ expr ~ ")" }
fweek = { ^"week" ~ "(" ~ expr ~ ")" }
fmonth = { ^"month" ~ "(" ~ expr ~ ")" }
fmonth_name = { ^"month_name" ~ "(" ~ expr ~ ")" }
fformat_time = { ^"format_time" ~ "(" ~ expr ~ "," ~ string_lit ~ ")" }
fparse_time = { ^"parse_time" ~ "(" ~ expr ~ "," ~ string_lit ~ ")" }
fquarter = { ^"quarter" ~ "(" ~ expr ~ ")" }
fyear = { ^"year" ~ "(" ~ expr ~ ")" }
DATE_PART = @{ ^"day_of_week" | ^"day" | ^"week" | ^"month" | ^"quarter" | ^"year" | ^"minute" | ^"hour" | ^"second" }
fdate_trunc = { (^"date_truncate" | ^"date_trunc") ~ "(" ~ expr ~ "," ~ DATE_PART ~ ")" }
fdate_part = { ^"date_part" ~ "(" ~ expr ~ "," ~ DATE_PART ~ ")" }
fdate_add = { ^"date_add" ~ "(" ~ expr ~ "," ~ DATE_PART ~ "," ~ expr ~ ")" }
fdate_sub = { ^"date_sub" ~ "(" ~ expr ~ "," ~ DATE_PART ~ "," ~ expr ~ ")" }
fdate_diff = { ^"date_diff" ~ "(" ~ expr ~ "," ~ expr ~ "," ~ DATE_PART ~ ")" }
fdate_spine = { ^"date_spine" ~ "(" ~ expr ~ "," ~ expr ~ ")" }
_date_functions = _{
fdate_trunc | fdate_part | fdate_add | fdate_sub | fdate_diff | fdate_spine
| fday_of_week | fday_name | fmonth_name | fformat_time | fparse_time
| fdate | fdatetime | ftimestamp | fsecond | fminute | fhour | fday | fweek | fmonth | fquarter | fyear
}
_static_functions = _{ _string_functions | _math_functions | _array_functions | _map_functions | _geo_functions }
custom_function = { "@" ~ IDENTIFIER ~ "(" ~ (expr ~ ("," ~ expr)* ~ ","?)? ~ ")" }
// =============================================================================
// LITERALS
// =============================================================================
concept_lit = { IDENTIFIER }
// int_lit / float_lit emit one nested token child so the v2 hydrator can
// `int(str(child))` / `float(str(child))`. Atomic inner token carries the span.
INT_LITERAL_PART = @{ "-"? ~ ASCII_DIGIT+ }
FLOAT_LITERAL_PART = @{ "-"? ~ ASCII_DIGIT* ~ "." ~ ASCII_DIGIT+ }
int_lit = ${ INT_LITERAL_PART ~ !("." ~ ASCII_DIGIT) ~ !(ASCII_ALPHA | "_") }
float_lit = ${ FLOAT_LITERAL_PART ~ !(ASCII_ALPHA | "_") }
// Keyword word-boundary: "null", "true", "false" must not prefix-match identifiers.
// Compound atomic `${}` captures inner named rules as pairs — the hydrator for
// bool_lit reads node.children[0], so we need BOOL_KW as a named inner token.
BOOL_KW = @{ ^"true" | ^"false" }
bool_lit = ${ BOOL_KW ~ !(ASCII_ALPHANUMERIC | "_") }
null_lit = ${ ^"null" ~ !(ASCII_ALPHANUMERIC | "_") }
array_lit = { "[" ~ (expr ~ ("," ~ expr)* ~ ","?)? ~ "]" }
tuple_lit = { "(" ~ (literal ~ ",")+ ~ literal? ~ ","? ~ ")" }
map_lit = { "{" ~ literal ~ ":" ~ literal ~ ("," ~ literal ~ ":" ~ literal)* ~ ","? ~ "}" }
struct_lit = { ^"struct" ~ "(" ~ expr ~ "->" ~ IDENTIFIER ~ ("," ~ expr ~ "->" ~ IDENTIFIER)* ~ ","? ~ ")" }
literal = { null_lit | string_lit | bool_lit | float_lit | int_lit | array_lit | map_lit | struct_lit | tuple_lit }
// Strings
MULTILINE_STRING = @{ "'''" ~ (!"'''" ~ ANY)* ~ "'''" }
DOUBLE_STRING_CHARS = @{ ((!"\"" ~ !"\\" ~ ANY) | ("\\" ~ ANY))+ }
SINGLE_STRING_CHARS = @{ ((!"'" ~ !"\\" ~ ANY) | ("\\" ~ ANY))+ }
string_lit = ${
MULTILINE_STRING
| ("'" ~ SINGLE_STRING_CHARS? ~ "'")
| ("\"" ~ DOUBLE_STRING_CHARS? ~ "\"")
}
// =============================================================================
// IDENTIFIERS + PATHS
// =============================================================================
IDENTIFIER = @{ !RESERVED_KW ~ (ASCII_ALPHA | "_") ~ (ASCII_ALPHANUMERIC | "_" | ".")* }
WILDCARD_IDENTIFIER = @{ (ASCII_ALPHA | "_") ~ (ASCII_ALPHANUMERIC | "_" | "-" | "." | "*")* }
QUOTED_IDENTIFIER = @{ "`" ~ (ASCII_ALPHA | "_") ~ (ASCII_ALPHANUMERIC | "_" | "." | "-" | "*" | ":" | " ")* ~ "`" }
QUOTED_ADDRESS = @{ "`" ~ "'"? ~ (ASCII_ALPHA | "_" | "\\" | "/") ~ (ASCII_ALPHANUMERIC | "_" | "." | "\\" | "/" | "-" | "*" | ":" | " ")* ~ "'"? ~ "`" }
F_QUOTED_ADDRESS = @{ "f`" ~ (!"`" ~ ANY)* ~ "`" }
ADDRESS = { IDENTIFIER }
// Reserved keywords that must not be parsed as identifiers. Limited to pure
// SQL clauses/operators that never appear as identifier names in trilogy code.
// `not` must be reserved so `concept_lit` can't absorb it as a bare
// identifier inside an _atom — otherwise `births ? not (a or b)` short
// circuits with concept_lit grabbing `not` and leaving `(a or b)`
// unconsumed, instead of falling through to `condition_parenthetical`.
// COMPARISON_OPERATOR / CONDITION_NOT / fnot all use
// literal `^"not"` matches, which aren't affected by this reservation.
RESERVED_KW = @{
(^"select" | ^"where" | ^"having" | ^"from" | ^"limit" | ^"and" | ^"or" | ^"not")
~ !(ASCII_ALPHANUMERIC | "_" | ".")
}
// Lookahead used at select_list / order_list boundaries to prevent greedy
// consumption of clause keywords after a trailing comma as an identifier.
ORDER_BY_LA = @{
^"order" ~ (" " | "\t" | "\r" | "\n")+ ~ ^"by"
~ !(ASCII_ALPHANUMERIC | "_" | ".")
}
SELECT_LIST_STOP = @{
( ORDER_BY_LA
| (^"merge" | ^"align" | ^"derive") ~ !(ASCII_ALPHANUMERIC | "_" | ".")
// a trailing comma before a post-select `<type> join` must not swallow the
// join keyword as another select item.
| JOIN_TYPE ~ (" " | "\t" | "\r" | "\n")+ ~ ^"join" ~ !(ASCII_ALPHANUMERIC | "_" | ".")
// a trailing comma before the SELECT-level `by rollup (…)` grouping clause
// must not swallow `by` as another select item (`by` is not reserved).
| ^"by" ~ (" " | "\t" | "\r" | "\n")+ ~ (^"rollup" | ^"cube" | ^"grouping") ~ !(ASCII_ALPHANUMERIC | "_" | ".")
)
}
FILE_EXT = _{ ^"py" | ^"csv" | ^"json" | ^"parquet" | ^"tsv" | ^"sql" }
FILE_PATH = @{ "`" ~ (!("." ~ FILE_EXT ~ "`") ~ !"`" ~ ANY)+ ~ "." ~ FILE_EXT ~ "`" }
F_FILE_PATH = @{ "f`" ~ (!("." ~ FILE_EXT ~ "`") ~ !"`" ~ ANY)+ ~ "." ~ FILE_EXT ~ "`" }
SHORTHAND_MODIFIER = @{ "~" | "?" }
JOIN_TYPE = @{ ^"left" | ^"inner" | ^"full" | ^"right" | ^"cross" | ^"subset" | ^"union" }
// =============================================================================
// PURPOSE / KEYWORDS (order matters: priority-like)
// =============================================================================
PURPOSE = @{ ^"key" | ^"metric" | ^"const" ~ ^"ant"? }
PROPERTY = @{ ^"property" }
UNIQUE = @{ ^"unique" }
AUTO = @{ ^"auto" }
// =============================================================================
// TYPES
// =============================================================================
numeric_type = { (^"numeric" | ^"decimal") ~ "(" ~ int_lit ~ "," ~ int_lit ~ ")" }
map_type = { ^"map" ~ "<" ~ (data_type | IDENTIFIER) ~ "," ~ (data_type | IDENTIFIER) ~ ">" }
struct_component = { IDENTIFIER ~ ":" ~ (validated_type | data_type) ~ concept_nullable_modifier? ~ metadata? }
struct_type = { ^"struct" ~ "<" ~ (struct_component | IDENTIFIER) ~ ("," ~ (struct_component | IDENTIFIER))* ~ ","? ~ ">" }
list_type = { (^"list" | ^"array") ~ "<" ~ (data_type | IDENTIFIER) ~ ">" }
enum_type = { ^"enum" ~ "<" ~ data_type ~ ">" ~ "[" ~ (string_lit | int_lit) ~ ("," ~ (string_lit | int_lit))* ~ ","? ~ "]" }
// Declaration-only validator suffix: int[0..100], float[0..,..1.5], string['[A-Z]+'],
// date['2020-01-01'..'2024-12-31']. Ranges are inclusive; comma = OR.
// Ordered before data_type at declaration sites so the bracket binds to the type.
RANGE_SEP = @{ ".." }
range_spec = { ((int_lit | float_lit | string_lit)? ~ RANGE_SEP ~ (int_lit | float_lit | string_lit)?) | (int_lit | float_lit | string_lit) }
VALIDATED_TYPE_BASE = @{
^"string" | ^"numeric" | ^"decimal" | ^"number" | ^"int" | ^"bigint"
| ^"datetime" | ^"timestamp" | ^"date" | ^"double" | ^"float"
}
validated_type = { (numeric_type | VALIDATED_TYPE_BASE) ~ "[" ~ range_spec ~ ("," ~ range_spec)* ~ ","? ~ "]" ~ ("::" ~ IDENTIFIER)? }
// data_type: include composite types before the simple keyword list (order matters).
data_type = {
(
numeric_type | map_type | struct_type | list_type | enum_type
| DATA_TYPE_SIMPLE
) ~ ("::" ~ IDENTIFIER)?
}
DATA_TYPE_SIMPLE = @{
^"string" | ^"bytes" | ^"geography" | ^"numeric" | ^"decimal" | ^"number" | ^"int" | ^"bigint"
| ^"datetime" | ^"timestamp" | ^"date" | ^"double" | ^"float" | ^"bool"
| ^"map" | ^"list" | ^"array" | ^"any"
}