sql_dialect_fmt_highlight/
lib.rs

1//! Lexical syntax highlighting for Snowflake SQL.
2//!
3//! This layer intentionally starts from the lossless lexer. It is stable under grammar growth:
4//! new keywords are highlighted by `keyword_kind`, while newly-added punctuation only needs a
5//! `SyntaxKind` classification here before LSP/TextMate adapters consume it.
6
7use sql_dialect_fmt_lexer::{tokenize, LexError};
8use sql_dialect_fmt_syntax::{is_builtin_type, keyword_kind, SyntaxKind};
9
10pub mod semantic;
11pub use semantic::{
12    delta_encode, detect_injections, line_tokens, resolve_tokens, semantic_token, semantic_tokens,
13    semantic_tokens_lsp, semantic_tokens_lsp_utf8, InjectedLanguage, Injection, LineToken,
14    ResolvedToken, SemanticTokenModifiers, SemanticTokenType, SemanticTokens,
15};
16
17/// The result of [`highlight`]: a lossless token stream plus any lexer errors. `#[non_exhaustive]`
18/// so fields can be added without breaking downstream matches.
19#[derive(Clone, Debug, PartialEq, Eq)]
20#[non_exhaustive]
21pub struct Highlighted<'a> {
22    /// Every token in source order; concatenating their `text` reproduces the input byte-for-byte.
23    pub tokens: Vec<HighlightToken<'a>>,
24    /// Lexer errors recovered while tokenizing (empty for clean input).
25    pub errors: Vec<LexError>,
26}
27
28/// A single highlighted token: a classified [`HighlightKind`] over a byte range of the source.
29/// `#[non_exhaustive]` so fields can be added without breaking downstream matches.
30#[derive(Clone, Debug, PartialEq, Eq)]
31#[non_exhaustive]
32pub struct HighlightToken<'a> {
33    /// The highlight classification of this token.
34    pub kind: HighlightKind,
35    /// The token's exact source text (the slice `&input[range]`).
36    pub text: &'a str,
37    /// Byte range of the token in the source.
38    pub range: std::ops::Range<usize>,
39}
40
41#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
42pub enum HighlightKind {
43    Whitespace,
44    Comment,
45    Keyword,
46    Type,
47    Identifier,
48    QuotedIdentifier,
49    String,
50    DollarString,
51    Number,
52    Variable,
53    Operator,
54    Punctuation,
55    Error,
56}
57
58impl HighlightKind {
59    pub const fn scope(self) -> &'static str {
60        match self {
61            HighlightKind::Whitespace => "text.whitespace",
62            HighlightKind::Comment => "comment.line-or-block",
63            HighlightKind::Keyword => "keyword.sql",
64            HighlightKind::Type => "support.type.sql",
65            HighlightKind::Identifier => "identifier.sql",
66            HighlightKind::QuotedIdentifier => "identifier.quoted.sql",
67            HighlightKind::String => "string.quoted.sql",
68            HighlightKind::DollarString => "string.dollar-quoted.sql",
69            HighlightKind::Number => "constant.numeric.sql",
70            HighlightKind::Variable => "variable.parameter.sql",
71            HighlightKind::Operator => "operator.sql",
72            HighlightKind::Punctuation => "punctuation.sql",
73            HighlightKind::Error => "invalid.illegal.sql",
74        }
75    }
76}
77
78pub fn highlight(input: &str) -> Highlighted<'_> {
79    let lexed = tokenize(input);
80    let tokens = lexed
81        .tokens
82        .into_iter()
83        .scan(0usize, |offset, token| {
84            let start = *offset;
85            *offset += token.text.len();
86            Some(HighlightToken {
87                kind: classify(token.kind, token.text),
88                text: token.text,
89                range: start..*offset,
90            })
91        })
92        .collect();
93
94    Highlighted {
95        tokens,
96        errors: lexed.errors,
97    }
98}
99
100pub fn classify(kind: SyntaxKind, text: &str) -> HighlightKind {
101    if kind.is_trivia() {
102        return match kind {
103            SyntaxKind::COMMENT | SyntaxKind::BLOCK_COMMENT => HighlightKind::Comment,
104            _ => HighlightKind::Whitespace,
105        };
106    }
107
108    if kind.is_keyword()
109        || kind == SyntaxKind::CONTEXTUAL_KEYWORD
110        || (kind == SyntaxKind::IDENT && keyword_kind(text).is_some())
111    {
112        return HighlightKind::Keyword;
113    }
114    if kind == SyntaxKind::IDENT && is_builtin_type(text) {
115        return HighlightKind::Type;
116    }
117
118    match kind {
119        SyntaxKind::IDENT => HighlightKind::Identifier,
120        SyntaxKind::QUOTED_IDENT => HighlightKind::QuotedIdentifier,
121        SyntaxKind::STRING => HighlightKind::String,
122        SyntaxKind::DOLLAR_STRING => HighlightKind::DollarString,
123        SyntaxKind::FILE_URI => HighlightKind::String,
124        SyntaxKind::INT_NUMBER | SyntaxKind::FLOAT_NUMBER => HighlightKind::Number,
125        SyntaxKind::VARIABLE | SyntaxKind::DOLLAR | SyntaxKind::QUESTION => HighlightKind::Variable,
126        SyntaxKind::ERROR | SyntaxKind::BANG => HighlightKind::Error,
127        kind if is_operator(kind) => HighlightKind::Operator,
128        _ => HighlightKind::Punctuation,
129    }
130}
131
132fn is_operator(kind: SyntaxKind) -> bool {
133    matches!(
134        kind,
135        SyntaxKind::COLON
136            | SyntaxKind::COLON2
137            | SyntaxKind::ASSIGN
138            | SyntaxKind::EQ
139            | SyntaxKind::NEQ
140            | SyntaxKind::LT
141            | SyntaxKind::LTE
142            | SyntaxKind::GT
143            | SyntaxKind::GTE
144            | SyntaxKind::PLUS
145            | SyntaxKind::MINUS
146            | SyntaxKind::STAR
147            | SyntaxKind::SLASH
148            | SyntaxKind::PERCENT
149            | SyntaxKind::CONCAT
150            | SyntaxKind::PIPE
151            | SyntaxKind::PIPE_GT
152            | SyntaxKind::FLOW_PIPE
153            | SyntaxKind::ARROW
154            | SyntaxKind::FAT_ARROW
155            | SyntaxKind::AMP
156            | SyntaxKind::CARET
157            | SyntaxKind::TILDE
158            | SyntaxKind::AT
159    )
160}
161
162#[cfg(test)]
163mod tests {
164    use super::*;
165
166    // The TextMate-grammar / highlighter consistency check now lives in `tests/textmate.rs`
167    // against the canonical `editors/snowflake.tmLanguage.json`. The old, superseded grammar
168    // under `editors/textmate/` (scope `source.sql.snowflake`) has been removed, so the
169    // `textmate_grammar_matches_the_highlighter` test that pinned it is gone with it.
170
171    #[test]
172    fn classifies_core_snowflake_tokens() {
173        let h = highlight("SELECT $1::NUMBER ->> SELECT \"name\" FROM $1 -- ok\n");
174        assert!(h.errors.is_empty());
175
176        let interesting: Vec<_> = h
177            .tokens
178            .iter()
179            .filter(|token| !matches!(token.kind, HighlightKind::Whitespace))
180            .map(|token| (token.text, token.kind))
181            .collect();
182
183        assert_eq!(
184            interesting,
185            vec![
186                ("SELECT", HighlightKind::Keyword),
187                ("$1", HighlightKind::Variable),
188                ("::", HighlightKind::Operator),
189                ("NUMBER", HighlightKind::Type),
190                ("->>", HighlightKind::Operator),
191                ("SELECT", HighlightKind::Keyword),
192                ("\"name\"", HighlightKind::QuotedIdentifier),
193                ("FROM", HighlightKind::Keyword),
194                ("$1", HighlightKind::Variable),
195                ("-- ok", HighlightKind::Comment),
196            ]
197        );
198    }
199
200    #[test]
201    fn ranges_cover_input_losslessly() {
202        let sql = "SELECT payload:items[0]::VARIANT FROM raw";
203        let highlighted = highlight(sql);
204        assert!(highlighted.errors.is_empty());
205        let joined: String = highlighted.tokens.iter().map(|token| token.text).collect();
206        assert_eq!(joined, sql);
207        for token in &highlighted.tokens {
208            assert_eq!(&sql[token.range.clone()], token.text);
209        }
210    }
211
212    #[test]
213    fn unquoted_file_uri_is_string_like_not_a_comment() {
214        let highlighted = highlight("PUT file:///tmp/data/mydata.csv @stage");
215        assert!(highlighted.errors.is_empty());
216        let uri = highlighted
217            .tokens
218            .iter()
219            .find(|token| token.text.starts_with("file://"))
220            .expect("a file URI token");
221        assert_eq!(uri.kind, HighlightKind::String);
222        assert!(!highlighted
223            .tokens
224            .iter()
225            .any(|token| token.kind == HighlightKind::Comment));
226    }
227}
sql_dialect_fmt_highlight/lib.rs

sql_dialect_fmt_highlight/
lib.rs
