sql_dialect_fmt_highlight/
lib.rs

1//! Lexical syntax highlighting for Snowflake SQL.
2//!
3//! This layer intentionally starts from the lossless lexer. It is stable under grammar growth:
4//! new keywords are highlighted by `keyword_kind`, while newly-added punctuation only needs a
5//! `SyntaxKind` classification here before LSP/TextMate adapters consume it.
6
7use sql_dialect_fmt_lexer::{tokenize, LexError};
8use sql_dialect_fmt_syntax::{keyword_kind, SyntaxKind};
9
10pub mod semantic;
11pub use semantic::{
12    delta_encode, detect_injections, line_tokens, resolve_tokens, semantic_token, semantic_tokens,
13    semantic_tokens_lsp, InjectedLanguage, Injection, LineToken, ResolvedToken,
14    SemanticTokenModifiers, SemanticTokenType, SemanticTokens,
15};
16
17/// The result of [`highlight`]: a lossless token stream plus any lexer errors. `#[non_exhaustive]`
18/// so fields can be added without breaking downstream matches.
19#[derive(Clone, Debug, PartialEq, Eq)]
20#[non_exhaustive]
21pub struct Highlighted<'a> {
22    /// Every token in source order; concatenating their `text` reproduces the input byte-for-byte.
23    pub tokens: Vec<HighlightToken<'a>>,
24    /// Lexer errors recovered while tokenizing (empty for clean input).
25    pub errors: Vec<LexError>,
26}
27
28/// A single highlighted token: a classified [`HighlightKind`] over a byte range of the source.
29/// `#[non_exhaustive]` so fields can be added without breaking downstream matches.
30#[derive(Clone, Debug, PartialEq, Eq)]
31#[non_exhaustive]
32pub struct HighlightToken<'a> {
33    /// The highlight classification of this token.
34    pub kind: HighlightKind,
35    /// The token's exact source text (the slice `&input[range]`).
36    pub text: &'a str,
37    /// Byte range of the token in the source.
38    pub range: std::ops::Range<usize>,
39}
40
41#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
42pub enum HighlightKind {
43    Whitespace,
44    Comment,
45    Keyword,
46    Type,
47    Identifier,
48    QuotedIdentifier,
49    String,
50    DollarString,
51    Number,
52    Variable,
53    Operator,
54    Punctuation,
55    Error,
56}
57
58impl HighlightKind {
59    pub const fn scope(self) -> &'static str {
60        match self {
61            HighlightKind::Whitespace => "text.whitespace",
62            HighlightKind::Comment => "comment.line-or-block",
63            HighlightKind::Keyword => "keyword.sql",
64            HighlightKind::Type => "support.type.sql",
65            HighlightKind::Identifier => "identifier.sql",
66            HighlightKind::QuotedIdentifier => "identifier.quoted.sql",
67            HighlightKind::String => "string.quoted.sql",
68            HighlightKind::DollarString => "string.dollar-quoted.sql",
69            HighlightKind::Number => "constant.numeric.sql",
70            HighlightKind::Variable => "variable.parameter.sql",
71            HighlightKind::Operator => "operator.sql",
72            HighlightKind::Punctuation => "punctuation.sql",
73            HighlightKind::Error => "invalid.illegal.sql",
74        }
75    }
76}
77
78pub fn highlight(input: &str) -> Highlighted<'_> {
79    let lexed = tokenize(input);
80    let tokens = lexed
81        .tokens
82        .into_iter()
83        .scan(0usize, |offset, token| {
84            let start = *offset;
85            *offset += token.text.len();
86            Some(HighlightToken {
87                kind: classify(token.kind, token.text),
88                text: token.text,
89                range: start..*offset,
90            })
91        })
92        .collect();
93
94    Highlighted {
95        tokens,
96        errors: lexed.errors,
97    }
98}
99
100pub fn classify(kind: SyntaxKind, text: &str) -> HighlightKind {
101    if kind.is_trivia() {
102        return match kind {
103            SyntaxKind::COMMENT | SyntaxKind::BLOCK_COMMENT => HighlightKind::Comment,
104            _ => HighlightKind::Whitespace,
105        };
106    }
107
108    if kind.is_keyword()
109        || kind == SyntaxKind::CONTEXTUAL_KEYWORD
110        || (kind == SyntaxKind::IDENT && keyword_kind(text).is_some())
111    {
112        return HighlightKind::Keyword;
113    }
114    if kind == SyntaxKind::IDENT && is_builtin_type(text) {
115        return HighlightKind::Type;
116    }
117
118    match kind {
119        SyntaxKind::IDENT => HighlightKind::Identifier,
120        SyntaxKind::QUOTED_IDENT => HighlightKind::QuotedIdentifier,
121        SyntaxKind::STRING => HighlightKind::String,
122        SyntaxKind::DOLLAR_STRING => HighlightKind::DollarString,
123        SyntaxKind::INT_NUMBER | SyntaxKind::FLOAT_NUMBER => HighlightKind::Number,
124        SyntaxKind::VARIABLE | SyntaxKind::DOLLAR | SyntaxKind::QUESTION => HighlightKind::Variable,
125        SyntaxKind::ERROR | SyntaxKind::BANG => HighlightKind::Error,
126        kind if is_operator(kind) => HighlightKind::Operator,
127        _ => HighlightKind::Punctuation,
128    }
129}
130
131fn is_builtin_type(text: &str) -> bool {
132    const TYPES: &[&str] = &[
133        "ARRAY",
134        "BIGINT",
135        "BINARY",
136        "BOOLEAN",
137        "CHAR",
138        "DATE",
139        "DATETIME",
140        "DEC",
141        "DECIMAL",
142        "DOUBLE",
143        "FLOAT",
144        "GEOGRAPHY",
145        "GEOMETRY",
146        "INT",
147        "INTEGER",
148        "MAP",
149        "NUMBER",
150        "NUMERIC",
151        "OBJECT",
152        "REAL",
153        "STRING",
154        "TEXT",
155        "TIME",
156        "TIMESTAMP",
157        "TIMESTAMP_LTZ",
158        "TIMESTAMP_NTZ",
159        "TIMESTAMP_TZ",
160        "VARIANT",
161        "VARCHAR",
162        "VECTOR",
163    ];
164    TYPES
165        .iter()
166        .any(|candidate| candidate.eq_ignore_ascii_case(text))
167}
168
169fn is_operator(kind: SyntaxKind) -> bool {
170    matches!(
171        kind,
172        SyntaxKind::COLON
173            | SyntaxKind::COLON2
174            | SyntaxKind::ASSIGN
175            | SyntaxKind::EQ
176            | SyntaxKind::NEQ
177            | SyntaxKind::LT
178            | SyntaxKind::LTE
179            | SyntaxKind::GT
180            | SyntaxKind::GTE
181            | SyntaxKind::PLUS
182            | SyntaxKind::MINUS
183            | SyntaxKind::STAR
184            | SyntaxKind::SLASH
185            | SyntaxKind::PERCENT
186            | SyntaxKind::CONCAT
187            | SyntaxKind::PIPE
188            | SyntaxKind::PIPE_GT
189            | SyntaxKind::FLOW_PIPE
190            | SyntaxKind::ARROW
191            | SyntaxKind::FAT_ARROW
192            | SyntaxKind::AMP
193            | SyntaxKind::CARET
194            | SyntaxKind::TILDE
195            | SyntaxKind::AT
196    )
197}
198
199#[cfg(test)]
200mod tests {
201    use super::*;
202
203    // The TextMate-grammar / highlighter consistency check now lives in `tests/textmate.rs`
204    // against the canonical `editors/snowflake.tmLanguage.json`. The old, superseded grammar
205    // under `editors/textmate/` (scope `source.sql.snowflake`) has been removed, so the
206    // `textmate_grammar_matches_the_highlighter` test that pinned it is gone with it.
207
208    #[test]
209    fn classifies_core_snowflake_tokens() {
210        let h = highlight("SELECT $1::NUMBER ->> SELECT \"name\" FROM $1 -- ok\n");
211        assert!(h.errors.is_empty());
212
213        let interesting: Vec<_> = h
214            .tokens
215            .iter()
216            .filter(|token| !matches!(token.kind, HighlightKind::Whitespace))
217            .map(|token| (token.text, token.kind))
218            .collect();
219
220        assert_eq!(
221            interesting,
222            vec![
223                ("SELECT", HighlightKind::Keyword),
224                ("$1", HighlightKind::Variable),
225                ("::", HighlightKind::Operator),
226                ("NUMBER", HighlightKind::Type),
227                ("->>", HighlightKind::Operator),
228                ("SELECT", HighlightKind::Keyword),
229                ("\"name\"", HighlightKind::QuotedIdentifier),
230                ("FROM", HighlightKind::Keyword),
231                ("$1", HighlightKind::Variable),
232                ("-- ok", HighlightKind::Comment),
233            ]
234        );
235    }
236
237    #[test]
238    fn ranges_cover_input_losslessly() {
239        let sql = "SELECT payload:items[0]::VARIANT FROM raw";
240        let highlighted = highlight(sql);
241        assert!(highlighted.errors.is_empty());
242        let joined: String = highlighted.tokens.iter().map(|token| token.text).collect();
243        assert_eq!(joined, sql);
244        for token in &highlighted.tokens {
245            assert_eq!(&sql[token.range.clone()], token.text);
246        }
247    }
248}
sql_dialect_fmt_highlight/lib.rs

sql_dialect_fmt_highlight/
lib.rs
