sql-dialect-fmt-parser 1.0.0

//! The parser state machine: lookahead, token matching, markers, and error recording.
//!
//! The grammar (in [`crate::grammar`]) drives this via a small vocabulary: `at`/`eat`/`bump`
//! for tokens, `start`/`complete`/`precede` for nodes. A `fuel` counter turns any accidental
//! non-advancing loop into a loud panic instead of a hang.

use std::cell::Cell;

use sql_dialect_fmt_syntax::{keyword_kind_for, Dialect, SyntaxKind};

use crate::event::Event;
use crate::input::Input;
use crate::ParseError;

const INITIAL_FUEL: u32 = 256;

/// Snowflake's *contextual keywords*: words that act as keywords only in a specific syntactic
/// position and otherwise remain ordinary identifiers. They are never lexed as keywords and never
/// reserved, so the grammar recognizes them by text via [`Parser::nth_contextual`]. Listing them in
/// one enum keeps the set discoverable and the match texts typo-proof (a misspelling is a compile
/// error rather than a silently-never-matching string).
macro_rules! contextual_keywords {
    ($(
        $(#[$meta:meta])*
        $variant:ident => $text:literal,
    )*) => {
        #[derive(Clone, Copy, PartialEq, Eq)]
        pub(crate) enum ContextualKeyword {
            $(
                $(#[$meta])*
                $variant,
            )*
        }

        impl ContextualKeyword {
            /// The lowercase source text this word matches case-insensitively.
            fn text(self) -> &'static str {
                match self {
                    $(
                        ContextualKeyword::$variant => $text,
                    )*
                }
            }
        }
    };
}

contextual_keywords! {
    /// `<table> AT (...)` — time travel.
    At => "at",
    /// `<table> BEFORE (...)` — time travel.
    Before => "before",
    /// `ASOF JOIN` — the join-type word.
    Asof => "asof",
    /// `ASOF JOIN ... MATCH_CONDITION (...)`.
    MatchCondition => "match_condition",
    /// `<table> MATCH_RECOGNIZE (...)`.
    MatchRecognize => "match_recognize",
    /// `GROUP BY GROUPING SETS (...)` — first word.
    Grouping => "grouping",
    /// `GROUPING SETS (...)` — second word.
    Sets => "sets",
    /// `MEASURES <expr> AS <alias> [, ...]`.
    Measures => "measures",
    /// `PATTERN ( <row pattern> )`.
    Pattern => "pattern",
    /// `DEFINE <symbol> AS <predicate> [, ...]`.
    Define => "define",
    /// `SUBSET <name> = ( <symbol>, ... )`.
    Subset => "subset",
    /// `... PER MATCH`, `AFTER MATCH SKIP`.
    Match => "match",
    /// `ONE ROW PER MATCH`.
    One => "one",
    /// `AFTER MATCH SKIP ...`.
    Skip => "skip",
    /// `AFTER MATCH SKIP PAST LAST ROW`.
    Past => "past",
    /// `AFTER MATCH SKIP TO NEXT ROW`.
    Next => "next",
    /// `AFTER MATCH SKIP TO [FIRST|LAST] <symbol>`.
    To => "to",
    /// `CONNECT BY NOCYCLE ...`.
    NoCycle => "nocycle",
    /// `<table> CHANGES ( INFORMATION => ... )` — change-tracking queries.
    Changes => "changes",
    /// `COMMENT ON <object> IS '...'` — recognized only before `ON` so it never shadows the very
    /// common `comment` column/identifier.
    Comment => "comment",
    /// Databricks table time travel: `VERSION AS OF <expr>`.
    Version => "version",
    /// Databricks table time travel: `TIMESTAMP AS OF <expr>`.
    Timestamp => "timestamp",
    /// The `OF` in Databricks `VERSION AS OF` / `TIMESTAMP AS OF`.
    Of => "of",
    /// Databricks `CREATE TABLE ... LOCATION '<path>'`.
    Location => "location",
    /// Databricks `CREATE TABLE ... TBLPROPERTIES (...)`.
    Tblproperties => "tblproperties",
    /// Databricks `CREATE TABLE ... OPTIONS (...)`.
    Options => "options",
    /// Databricks `CREATE TABLE ... PARTITIONED BY (...)`.
    Partitioned => "partitioned",
    /// `BEGIN TRANSACTION` — distinguishes a transaction start from a Snowflake Scripting block.
    Transaction => "transaction",
    /// `BEGIN WORK` — the SQL-standard spelling of a transaction start.
    Work => "work",
    /// `FOR i IN REVERSE <start> TO <end>` — counts down.
    Reverse => "reverse",
    /// `<name> [<type>] DEFAULT <expr>` — a declaration's default value (also a DDL column default).
    Default => "default",
    /// `BREAK [<label>]` — exit a loop.
    Break => "break",
    /// `CONTINUE [<label>]` — skip to the next loop iteration.
    Continue => "continue",
    /// `CREATE SCHEMA …`.
    Schema => "schema",
    /// `CREATE DATABASE …`.
    Database => "database",
    /// `CREATE STAGE …` / `… ON STAGE …`.
    Stage => "stage",
    /// `CREATE SEQUENCE …`.
    Sequence => "sequence",
    /// `CREATE STREAM …`.
    Stream => "stream",
    /// `CREATE DYNAMIC TABLE …`.
    Dynamic => "dynamic",
    /// `CREATE SEMANTIC VIEW …`.
    Semantic => "semantic",
    /// `CREATE FILE FORMAT …`.
    File => "file",
    /// `CREATE FILE FORMAT …` — second word.
    Format => "format",
    /// `CREATE MASKING POLICY …`.
    Masking => "masking",
    /// `CREATE MASKING POLICY …` / `CREATE ROW ACCESS POLICY …`.
    Policy => "policy",
    /// `CREATE ROW ACCESS POLICY …` — middle word.
    Access => "access",
    /// `CREATE TAG …`.
    Tag => "tag",
    /// `CREATE TAG … ALLOWED_VALUES ...`.
    AllowedValues => "allowed_values",
    /// `CREATE TAG … PROPAGATE = ...`.
    Propagate => "propagate",
    /// `CREATE MASKING POLICY … EXEMPT_OTHER_POLICIES = ...`.
    ExemptOtherPolicies => "exempt_other_policies",
    /// `CREATE SEMANTIC VIEW … WITH TABLES`.
    Tables => "tables",
    /// `CREATE SEMANTIC VIEW … WITH RELATIONSHIPS`.
    Relationships => "relationships",
    /// `CREATE SEMANTIC VIEW … WITH FACTS`.
    Facts => "facts",
    /// `CREATE SEMANTIC VIEW … WITH DIMENSIONS`.
    Dimensions => "dimensions",
    /// `CREATE SEMANTIC VIEW … WITH METRICS`.
    Metrics => "metrics",
    /// `CREATE SEMANTIC VIEW … PUBLIC`.
    Public => "public",
    /// `CREATE SEMANTIC VIEW … PRIVATE`.
    Private => "private",
    /// Semantic View `SYNONYMS` attribute.
    Synonyms => "synonyms",
    /// Semantic View `LABELS` attribute.
    Labels => "labels",
    /// Semantic View `AI_SQL_GENERATION` attribute.
    AiSqlGeneration => "ai_sql_generation",
    /// Semantic View `AI_QUESTION_CATEGORIZATION` attribute.
    AiQuestionCategorization => "ai_question_categorization",
    /// Semantic View `AI_VERIFIED_QUERIES` attribute.
    AiVerifiedQueries => "ai_verified_queries",
    /// Semantic View verified-query `QUESTION`.
    Question => "question",
    /// Semantic View verified-query `VERIFIED_AT`.
    VerifiedAt => "verified_at",
    /// Semantic View verified-query `ONBOARDING_QUESTION`.
    OnboardingQuestion => "onboarding_question",
    /// Semantic View verified-query `VERIFIED_BY`.
    VerifiedBy => "verified_by",
    /// `… TO ROLE r` / `… FROM ROLE r`.
    Role => "role",
    /// `… TO USER u`.
    User => "user",
    /// `GRANT <role> TO SHARE s`.
    Share => "share",
    /// `REVOKE … FROM r RESTRICT|CASCADE` — cascade.
    Cascade => "cascade",
    /// `REVOKE … FROM r RESTRICT` — restrict.
    Restrict => "restrict",
    /// `REVOKE GRANT OPTION FOR …` / `… WITH GRANT OPTION` — option.
    Option => "option",
    /// `GRANT ALL PRIVILEGES …`.
    Privileges => "privileges",
    /// `MATERIALIZED VIEW`.
    Materialized => "materialized",
    /// `LOCAL TEMP[ORARY]` table/view modifier.
    Local => "local",
    /// `GLOBAL TEMP[ORARY]` table/view modifier.
    Global => "global",
    /// `CLUSTER BY ( ... )`.
    Cluster => "cluster",
    /// `CREATE TABLE <name> CLONE <source>`.
    Clone => "clone",
    /// A `PRIMARY KEY` constraint (also the first word of the two).
    Primary => "primary",
    /// The `KEY` of `PRIMARY KEY` / `FOREIGN KEY`.
    Key => "key",
    /// A `UNIQUE` constraint.
    Unique => "unique",
    /// A `FOREIGN KEY` constraint.
    Foreign => "foreign",
    /// `FOREIGN KEY ( ... ) REFERENCES <table> ( ... )`.
    References => "references",
    /// `CONSTRAINT <name> ...` — names an out-of-line constraint.
    Constraint => "constraint",
    /// A `CHECK ( <expr> )` constraint.
    Check => "check",
    /// A column `COLLATE '<spec>'`.
    Collate => "collate",
    // ---- Databricks / Delta maintenance + cache statements (contextual at statement start, so the
    // words stay ordinary identifiers under Snowflake and elsewhere under Databricks) ----
    /// `VACUUM <table|path> …` — Delta file cleanup statement.
    Vacuum => "vacuum",
    /// `VACUUM <t> RETAIN <n> HOURS …`.
    Retain => "retain",
    /// `VACUUM <t> RETAIN <n> HOURS` — the `HOURS` unit word.
    Hours => "hours",
    /// `VACUUM <t> … DRY RUN` — first word.
    Dry => "dry",
    /// `VACUUM <t> … DRY RUN` — second word.
    Run => "run",
    /// `OPTIMIZE <table> [WHERE p] [ZORDER BY (cols)]` — Delta compaction statement.
    Optimize => "optimize",
    /// `OPTIMIZE <t> ZORDER BY ( col [, ...] )` — the z-order clause keyword.
    Zorder => "zorder",
    /// `CACHE [LAZY] TABLE <t> …` — Spark cache statement.
    Cache => "cache",
    /// `CACHE LAZY TABLE …` — defer caching until first use.
    Lazy => "lazy",
    /// `UNCACHE TABLE [IF EXISTS] <t>` — drop a cached table.
    Uncache => "uncache",
    /// `REFRESH [TABLE] <t>` / `REFRESH <path>` — invalidate cached entries.
    Refresh => "refresh",
    /// `DESCRIBE HISTORY <table>` — Delta change-history statement (the `HISTORY` word).
    History => "history",
    /// `MERGE … WHEN NOT MATCHED BY SOURCE …` — the `SOURCE` qualifier word.
    Source => "source",
    /// `MERGE … WHEN NOT MATCHED BY TARGET …` — the `TARGET` qualifier word.
    Target => "target",
}

pub(crate) struct Parser<'a> {
    input: &'a Input<'a>,
    dialect: Dialect,
    pos: usize,
    fuel: Cell<u32>,
    events: Vec<Event>,
    errors: Vec<ParseError>,
}

impl<'a> Parser<'a> {
    pub(crate) fn new(input: &'a Input<'a>, dialect: Dialect) -> Self {
        Parser {
            input,
            dialect,
            pos: 0,
            fuel: Cell::new(INITIAL_FUEL),
            events: Vec::new(),
            errors: Vec::new(),
        }
    }

    /// The SQL dialect this parse targets. Grammar branches that recognize dialect-specific
    /// constructs gate on this via the [`Dialect`] predicate methods.
    pub(crate) fn dialect(&self) -> Dialect {
        self.dialect
    }

    pub(crate) fn parse(mut self) -> (Vec<Event>, Vec<ParseError>) {
        crate::grammar::source_file(&mut self);
        (self.events, self.errors)
    }

    // ---- lookahead ----

    pub(crate) fn at_eof(&self) -> bool {
        self.pos >= self.input.len()
    }

    pub(crate) fn pos(&self) -> usize {
        self.pos
    }

    /// Raw kind `n` tokens ahead, consuming a unit of fuel (the loop guard).
    fn nth(&self, n: usize) -> SyntaxKind {
        if self.pos + n >= self.input.len() {
            return SyntaxKind::EOF;
        }
        assert_ne!(
            self.fuel.get(),
            0,
            "parser stuck — no progress at pos {}",
            self.pos
        );
        self.fuel.set(self.fuel.get() - 1);
        self.input.kind(self.pos + n)
    }

    /// Whether `text` is a *reserved* keyword in this parser's [`Dialect`], and its kind if so.
    ///
    /// The single point where keyword reservation is resolved for the active dialect: every
    /// keyword decision (`at`, `at_name`, `at_keyword`, `nth_at`, `current_remapped`) routes through
    /// here, so a Snowflake-only word like `TASK`/`FLATTEN` is a plain identifier under Databricks
    /// while Snowflake reservation (the default) is unchanged.
    #[inline]
    fn keyword_kind(&self, text: &str) -> Option<SyntaxKind> {
        keyword_kind_for(text, self.dialect)
    }

    /// Is the current token `kind`? Keyword kinds match a raw `IDENT` whose text is that keyword.
    pub(crate) fn at(&self, kind: SyntaxKind) -> bool {
        if kind.is_keyword() {
            self.nth(0) == SyntaxKind::IDENT
                && self.keyword_kind(self.input.text(self.pos)) == Some(kind)
        } else {
            self.nth(0) == kind
        }
    }

    /// Is the current token a usable *name* (a non-keyword identifier or a quoted identifier)?
    pub(crate) fn at_name(&self) -> bool {
        match self.nth(0) {
            SyntaxKind::QUOTED_IDENT => true,
            SyntaxKind::IDENT => self
                .keyword_kind(self.input.text(self.pos))
                .is_none_or(is_identifier_compatible_keyword),
            _ => false,
        }
    }

    /// Is the current token identifier-like (a bare `IDENT`, keyword or not, or a quoted
    /// identifier)? Used to recognize a named-argument label before `=>`.
    pub(crate) fn at_ident_like(&self) -> bool {
        matches!(self.nth(0), SyntaxKind::IDENT | SyntaxKind::QUOTED_IDENT)
    }

    /// Is the current token a (reserved-spelled) keyword word? Used to recognize a keyword used as a
    /// function name (`first(x)`, `last(x)`), the complement of [`Self::at_name`].
    pub(crate) fn at_keyword(&self) -> bool {
        self.nth(0) == SyntaxKind::IDENT && self.keyword_kind(self.input.text(self.pos)).is_some()
    }

    /// Is the token `n` ahead a given [`ContextualKeyword`]: a bare `IDENT` whose text matches
    /// case-insensitively? Used for non-reserved words like `GROUPING`/`SETS` that must not become
    /// real keywords (they double as the `GROUPING(col)` function and ordinary identifiers).
    pub(crate) fn nth_contextual(&self, n: usize, kw: ContextualKeyword) -> bool {
        self.nth(n) == SyntaxKind::IDENT
            && self
                .input
                .text(self.pos + n)
                .eq_ignore_ascii_case(kw.text())
    }

    /// Like [`Self::at`], but `n` tokens ahead — used for the handful of two-token decisions
    /// (`NOT IN`, `NOT LIKE`, `( SELECT`, ...).
    pub(crate) fn nth_at(&self, n: usize, kind: SyntaxKind) -> bool {
        if kind.is_keyword() {
            self.nth(n) == SyntaxKind::IDENT
                && self.keyword_kind(self.input.text(self.pos + n)) == Some(kind)
        } else {
            self.nth(n) == kind
        }
    }

    /// The kind to tag the current token with: a keyword kind if the raw `IDENT` is a keyword,
    /// otherwise the raw kind. Keeps keyword tokens correctly typed in the tree for highlighting.
    fn current_remapped(&self) -> SyntaxKind {
        let raw = self.input.kind(self.pos);
        if raw == SyntaxKind::IDENT {
            self.keyword_kind(self.input.text(self.pos))
                .unwrap_or(SyntaxKind::IDENT)
        } else {
            raw
        }
    }

    // ---- consumption ----

    fn advance(&mut self, kind: SyntaxKind) {
        // Never advance past end of input. Callers guard with `at`/`!at_eof`, but stay total
        // (return rather than `assert!`) so a stray call at EOF can never panic — the parser's
        // hard never-panic invariant. `bump`/`bump_as` keep their `debug_assert` guards for tests.
        if self.at_eof() {
            return;
        }
        self.events.push(Event::Advance { kind });
        self.pos += 1;
        self.fuel.set(INITIAL_FUEL);
    }

    /// Consume the current token, tagging keywords with their keyword kind.
    pub(crate) fn bump_any(&mut self) {
        let kind = self.current_remapped();
        self.advance(kind);
    }

    /// Consume the current token, tagging it with `kind` regardless of its keyword-ness. Used for
    /// positions where a keyword-spelled word is really a plain identifier (e.g. a case-sensitive
    /// semi-structured path key like `payload:order`), so it is not later up-cased as a keyword.
    pub(crate) fn bump_as(&mut self, kind: SyntaxKind) {
        debug_assert!(!self.at_eof(), "bump_as past end of input");
        self.advance(kind);
    }

    /// Consume the current token, asserting it is `kind` (used after an `at` check).
    pub(crate) fn bump(&mut self, kind: SyntaxKind) {
        debug_assert!(self.at(kind), "bump({kind:?}) but not at it");
        self.advance(kind);
    }

    /// Consume the current token iff it is `kind`.
    pub(crate) fn eat(&mut self, kind: SyntaxKind) -> bool {
        if self.at(kind) {
            self.advance(kind);
            true
        } else {
            false
        }
    }

    /// Consume `kind` if present, else record a diagnostic (no token is consumed). The message
    /// names the expected token with its human-readable spelling (`expected INTO`, `expected '('`),
    /// never the internal `SyntaxKind` debug name.
    pub(crate) fn expect(&mut self, kind: SyntaxKind) {
        if !self.eat(kind) {
            self.error(format!("expected {}", kind.describe()));
        }
    }

    // ---- errors ----

    /// Record a diagnostic spanning the current (offending) token. At end of input the span is the
    /// zero-width point at the source's end, so the message still attaches to a location.
    pub(crate) fn error(&mut self, msg: impl Into<String>) {
        let offset = self.input.offset(self.pos);
        let len = self.input.token_len(self.pos);
        self.errors.push(ParseError {
            message: msg.into(),
            offset,
            len,
        });
    }

    /// Record a diagnostic and wrap the offending token in an `ERROR` node so the tree stays
    /// lossless and parsing keeps making progress.
    pub(crate) fn err_and_bump(&mut self, msg: impl Into<String>) {
        self.error(msg);
        if self.at_eof() {
            return;
        }
        let m = self.start();
        self.bump_any();
        m.complete(self, SyntaxKind::ERROR);
    }

    // ---- markers ----

    pub(crate) fn start(&mut self) -> Marker {
        let index = self.events.len();
        self.events.push(Event::Open {
            kind: SyntaxKind::ERROR,
        });
        Marker {
            index,
            completed: false,
        }
    }
}

fn is_identifier_compatible_keyword(kind: SyntaxKind) -> bool {
    matches!(
        kind,
        SyntaxKind::LANGUAGE_KW
            | SyntaxKind::JAVASCRIPT_KW
            | SyntaxKind::PYTHON_KW
            | SyntaxKind::JAVA_KW
            | SyntaxKind::SCALA_KW
            | SyntaxKind::SQL_KW
    )
}

/// A still-open node. Must be `complete`d (a `DropBomb` catches grammar bugs that forget to).
pub(crate) struct Marker {
    index: usize,
    completed: bool,
}

impl Marker {
    pub(crate) fn complete(mut self, p: &mut Parser, kind: SyntaxKind) -> CompletedMarker {
        self.completed = true;
        p.events[self.index] = Event::Open { kind };
        p.events.push(Event::Close);
        CompletedMarker { index: self.index }
    }

    /// Discard this (speculative) wrapper: its `Open` becomes a no-op and any nodes parsed inside it
    /// stay attached to the parent. Used when a wrapper is started before knowing whether it is
    /// needed (e.g. a flow-operator chain that turns out to be a single statement).
    pub(crate) fn abandon(mut self, p: &mut Parser) {
        self.completed = true;
        p.events[self.index] = Event::Tombstone;
    }
}

impl Drop for Marker {
    fn drop(&mut self) {
        if !self.completed && !std::thread::panicking() {
            panic!("Marker dropped without being completed");
        }
    }
}

/// A finished node, which can be retroactively wrapped by a new parent via [`Self::precede`].
#[derive(Clone, Copy)]
pub(crate) struct CompletedMarker {
    index: usize,
}

impl CompletedMarker {
    /// Start a new node that begins where this one began (left-associative wrapping, e.g. for
    /// binary expressions). Inserts an `Open` before this node's `Open`.
    pub(crate) fn precede(self, p: &mut Parser) -> Marker {
        p.events.insert(
            self.index,
            Event::Open {
                kind: SyntaxKind::ERROR,
            },
        );
        Marker {
            index: self.index,
            completed: false,
        }
    }
}