sql_dialect_fmt_highlight/
lib.rs

1//! Lexical syntax highlighting for Snowflake SQL.
2//!
3//! This layer intentionally starts from the lossless lexer. It is stable under grammar growth:
4//! new keywords are highlighted by `keyword_kind`, while newly-added punctuation only needs a
5//! `SyntaxKind` classification here before LSP/TextMate adapters consume it.
6
7use sql_dialect_fmt_lexer::{tokenize, LexError};
8use sql_dialect_fmt_syntax::{is_builtin_type, keyword_kind, SyntaxKind};
9
10pub mod semantic;
11pub use semantic::{
12    delta_encode, detect_injections, line_tokens, resolve_tokens, semantic_token, semantic_tokens,
13    semantic_tokens_lsp, semantic_tokens_lsp_utf8, InjectedLanguage, Injection, LineToken,
14    ResolvedToken, SemanticTokenModifiers, SemanticTokenType, SemanticTokens,
15};
16
17/// The result of [`highlight`]: a lossless token stream plus any lexer errors. `#[non_exhaustive]`
18/// so fields can be added without breaking downstream matches.
19#[derive(Clone, Debug, PartialEq, Eq)]
20#[non_exhaustive]
21pub struct Highlighted<'a> {
22    /// Every token in source order; concatenating their `text` reproduces the input byte-for-byte.
23    pub tokens: Vec<HighlightToken<'a>>,
24    /// Lexer errors recovered while tokenizing (empty for clean input).
25    pub errors: Vec<LexError>,
26}
27
28/// A single highlighted token: a classified [`HighlightKind`] over a byte range of the source.
29/// `#[non_exhaustive]` so fields can be added without breaking downstream matches.
30#[derive(Clone, Debug, PartialEq, Eq)]
31#[non_exhaustive]
32pub struct HighlightToken<'a> {
33    /// The highlight classification of this token.
34    pub kind: HighlightKind,
35    /// The token's exact source text (the slice `&input[range]`).
36    pub text: &'a str,
37    /// Byte range of the token in the source.
38    pub range: std::ops::Range<usize>,
39}
40
41#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
42pub enum HighlightKind {
43    Whitespace,
44    Comment,
45    Keyword,
46    Type,
47    Identifier,
48    QuotedIdentifier,
49    String,
50    DollarString,
51    Number,
52    Variable,
53    Operator,
54    Punctuation,
55    Error,
56}
57
58impl HighlightKind {
59    pub const fn scope(self) -> &'static str {
60        match self {
61            HighlightKind::Whitespace => "text.whitespace",
62            HighlightKind::Comment => "comment.line-or-block",
63            HighlightKind::Keyword => "keyword.sql",
64            HighlightKind::Type => "support.type.sql",
65            HighlightKind::Identifier => "identifier.sql",
66            HighlightKind::QuotedIdentifier => "identifier.quoted.sql",
67            HighlightKind::String => "string.quoted.sql",
68            HighlightKind::DollarString => "string.dollar-quoted.sql",
69            HighlightKind::Number => "constant.numeric.sql",
70            HighlightKind::Variable => "variable.parameter.sql",
71            HighlightKind::Operator => "operator.sql",
72            HighlightKind::Punctuation => "punctuation.sql",
73            HighlightKind::Error => "invalid.illegal.sql",
74        }
75    }
76}
77
78pub fn highlight(input: &str) -> Highlighted<'_> {
79    let lexed = tokenize(input);
80    let tokens = lexed
81        .tokens
82        .into_iter()
83        .scan(0usize, |offset, token| {
84            let start = *offset;
85            *offset += token.text.len();
86            Some(HighlightToken {
87                kind: classify(token.kind, token.text),
88                text: token.text,
89                range: start..*offset,
90            })
91        })
92        .collect();
93
94    Highlighted {
95        tokens,
96        errors: lexed.errors,
97    }
98}
99
100pub fn classify(kind: SyntaxKind, text: &str) -> HighlightKind {
101    if kind.is_trivia() {
102        return match kind {
103            SyntaxKind::COMMENT | SyntaxKind::BLOCK_COMMENT => HighlightKind::Comment,
104            _ => HighlightKind::Whitespace,
105        };
106    }
107
108    if kind.is_keyword()
109        || kind == SyntaxKind::CONTEXTUAL_KEYWORD
110        || (kind == SyntaxKind::IDENT && keyword_kind(text).is_some())
111    {
112        return HighlightKind::Keyword;
113    }
114    if kind == SyntaxKind::IDENT && is_builtin_type(text) {
115        return HighlightKind::Type;
116    }
117
118    match kind {
119        SyntaxKind::IDENT => HighlightKind::Identifier,
120        SyntaxKind::QUOTED_IDENT => HighlightKind::QuotedIdentifier,
121        SyntaxKind::STRING => HighlightKind::String,
122        SyntaxKind::DOLLAR_STRING => HighlightKind::DollarString,
123        SyntaxKind::FILE_URI => HighlightKind::String,
124        SyntaxKind::INT_NUMBER | SyntaxKind::FLOAT_NUMBER => HighlightKind::Number,
125        SyntaxKind::VARIABLE
126        | SyntaxKind::PLACEHOLDER
127        | SyntaxKind::DOLLAR
128        | SyntaxKind::QUESTION => HighlightKind::Variable,
129        SyntaxKind::ERROR | SyntaxKind::BANG => HighlightKind::Error,
130        kind if is_operator(kind) => HighlightKind::Operator,
131        _ => HighlightKind::Punctuation,
132    }
133}
134
135fn is_operator(kind: SyntaxKind) -> bool {
136    matches!(
137        kind,
138        SyntaxKind::COLON
139            | SyntaxKind::COLON2
140            | SyntaxKind::ASSIGN
141            | SyntaxKind::EQ
142            | SyntaxKind::NEQ
143            | SyntaxKind::LT
144            | SyntaxKind::LTE
145            | SyntaxKind::GT
146            | SyntaxKind::GTE
147            | SyntaxKind::PLUS
148            | SyntaxKind::MINUS
149            | SyntaxKind::STAR
150            | SyntaxKind::SLASH
151            | SyntaxKind::PERCENT
152            | SyntaxKind::CONCAT
153            | SyntaxKind::PIPE
154            | SyntaxKind::PIPE_GT
155            | SyntaxKind::FLOW_PIPE
156            | SyntaxKind::ARROW
157            | SyntaxKind::FAT_ARROW
158            | SyntaxKind::AMP
159            | SyntaxKind::CARET
160            | SyntaxKind::TILDE
161            | SyntaxKind::AT
162    )
163}
164
165#[cfg(test)]
166mod tests {
167    use super::*;
168
169    // The TextMate-grammar / highlighter consistency check now lives in `tests/textmate.rs`
170    // against the canonical `editors/snowflake.tmLanguage.json`. The old, superseded grammar
171    // under `editors/textmate/` (scope `source.sql.snowflake`) has been removed, so the
172    // `textmate_grammar_matches_the_highlighter` test that pinned it is gone with it.
173
174    #[test]
175    fn classifies_core_snowflake_tokens() {
176        let h = highlight("SELECT $1::NUMBER ->> SELECT \"name\" FROM $1 -- ok\n");
177        assert!(h.errors.is_empty());
178
179        let interesting: Vec<_> = h
180            .tokens
181            .iter()
182            .filter(|token| !matches!(token.kind, HighlightKind::Whitespace))
183            .map(|token| (token.text, token.kind))
184            .collect();
185
186        assert_eq!(
187            interesting,
188            vec![
189                ("SELECT", HighlightKind::Keyword),
190                ("$1", HighlightKind::Variable),
191                ("::", HighlightKind::Operator),
192                ("NUMBER", HighlightKind::Type),
193                ("->>", HighlightKind::Operator),
194                ("SELECT", HighlightKind::Keyword),
195                ("\"name\"", HighlightKind::QuotedIdentifier),
196                ("FROM", HighlightKind::Keyword),
197                ("$1", HighlightKind::Variable),
198                ("-- ok", HighlightKind::Comment),
199            ]
200        );
201    }
202
203    #[test]
204    fn ranges_cover_input_losslessly() {
205        let sql = "SELECT payload:items[0]::VARIANT FROM raw";
206        let highlighted = highlight(sql);
207        assert!(highlighted.errors.is_empty());
208        let joined: String = highlighted.tokens.iter().map(|token| token.text).collect();
209        assert_eq!(joined, sql);
210        for token in &highlighted.tokens {
211            assert_eq!(&sql[token.range.clone()], token.text);
212        }
213    }
214
215    #[test]
216    fn unquoted_file_uri_is_string_like_not_a_comment() {
217        let highlighted = highlight("PUT file:///tmp/data/mydata.csv @stage");
218        assert!(highlighted.errors.is_empty());
219        let uri = highlighted
220            .tokens
221            .iter()
222            .find(|token| token.text.starts_with("file://"))
223            .expect("a file URI token");
224        assert_eq!(uri.kind, HighlightKind::String);
225        assert!(!highlighted
226            .tokens
227            .iter()
228            .any(|token| token.kind == HighlightKind::Comment));
229    }
230}
sql_dialect_fmt_highlight/lib.rs

sql_dialect_fmt_highlight/
lib.rs
