squawk_parser/
lexed_str.rs

1// based on https://github.com/rust-lang/rust-analyzer/blob/d8887c0758bbd2d5f752d5bd405d4491e90e7ed6/crates/parser/src/lexed_str.rs
2
3use std::ops;
4
5use squawk_lexer::tokenize;
6
7use crate::SyntaxKind;
8
9pub struct LexedStr<'a> {
10    text: &'a str,
11    kind: Vec<SyntaxKind>,
12    start: Vec<u32>,
13    error: Vec<LexError>,
14}
15
16struct LexError {
17    msg: String,
18    token: u32,
19}
20
21impl<'a> LexedStr<'a> {
22    // TODO: rust-analyzer has an edition thing to specify things that are only
23    // available in certain version, we can do that later
24    pub fn new(text: &'a str) -> LexedStr<'a> {
25        let mut conv = Converter::new(text);
26
27        for token in tokenize(&text[conv.offset..]) {
28            let token_text = &text[conv.offset..][..token.len as usize];
29
30            conv.extend_token(&token.kind, token_text);
31        }
32
33        conv.finalize_with_eof()
34    }
35
36    // pub(crate) fn single_token(text: &'a str) -> Option<(SyntaxKind, Option<String>)> {
37    //     if text.is_empty() {
38    //         return None;
39    //     }
40
41    //     let token = tokenize(text).next()?;
42    //     if token.len as usize != text.len() {
43    //         return None;
44    //     }
45
46    //     let mut conv = Converter::new(text);
47    //     conv.extend_token(&token.kind, text);
48    //     match &*conv.res.kind {
49    //         [kind] => Some((*kind, conv.res.error.pop().map(|it| it.msg))),
50    //         _ => None,
51    //     }
52    // }
53
54    // pub(crate) fn as_str(&self) -> &str {
55    //     self.text
56    // }
57
58    pub(crate) fn len(&self) -> usize {
59        self.kind.len() - 1
60    }
61
62    // pub(crate) fn is_empty(&self) -> bool {
63    //     self.len() == 0
64    // }
65
66    pub(crate) fn kind(&self, i: usize) -> SyntaxKind {
67        assert!(i < self.len());
68        self.kind[i]
69    }
70
71    pub(crate) fn text(&self, i: usize) -> &str {
72        self.range_text(i..i + 1)
73    }
74
75    pub(crate) fn range_text(&self, r: ops::Range<usize>) -> &str {
76        assert!(r.start < r.end && r.end <= self.len());
77        let lo = self.start[r.start] as usize;
78        let hi = self.start[r.end] as usize;
79        &self.text[lo..hi]
80    }
81
82    // Naming is hard.
83    pub fn text_range(&self, i: usize) -> ops::Range<usize> {
84        assert!(i < self.len());
85        let lo = self.start[i] as usize;
86        let hi = self.start[i + 1] as usize;
87        lo..hi
88    }
89    pub fn text_start(&self, i: usize) -> usize {
90        assert!(i <= self.len());
91        self.start[i] as usize
92    }
93    // pub(crate) fn text_len(&self, i: usize) -> usize {
94    //     assert!(i < self.len());
95    //     let r = self.text_range(i);
96    //     r.end - r.start
97    // }
98
99    // pub(crate) fn error(&self, i: usize) -> Option<&str> {
100    //     assert!(i < self.len());
101    //     let err = self
102    //         .error
103    //         .binary_search_by_key(&(i as u32), |i| i.token)
104    //         .ok()?;
105    //     Some(self.error[err].msg.as_str())
106    // }
107
108    pub fn errors(&self) -> impl Iterator<Item = (usize, &str)> + '_ {
109        self.error
110            .iter()
111            .map(|it| (it.token as usize, it.msg.as_str()))
112    }
113
114    fn push(&mut self, kind: SyntaxKind, offset: usize) {
115        self.kind.push(kind);
116        self.start.push(offset as u32);
117    }
118}
119
120struct Converter<'a> {
121    res: LexedStr<'a>,
122    offset: usize,
123}
124
125impl<'a> Converter<'a> {
126    fn new(text: &'a str) -> Self {
127        Self {
128            res: LexedStr {
129                text,
130                kind: Vec::new(),
131                start: Vec::new(),
132                error: Vec::new(),
133            },
134            offset: 0,
135        }
136    }
137
138    fn finalize_with_eof(mut self) -> LexedStr<'a> {
139        self.res.push(SyntaxKind::EOF, self.offset);
140        self.res
141    }
142
143    fn push(&mut self, kind: SyntaxKind, len: usize, err: Option<&str>) {
144        self.res.push(kind, self.offset);
145        self.offset += len;
146
147        if let Some(err) = err {
148            let token = self.res.len() as u32;
149            let msg = err.to_owned();
150            self.res.error.push(LexError { msg, token });
151        }
152    }
153
154    fn extend_token(&mut self, kind: &squawk_lexer::TokenKind, token_text: &str) {
155        // A note on an intended tradeoff:
156        // We drop some useful information here (see patterns with double dots `..`)
157        // Storing that info in `SyntaxKind` is not possible due to its layout requirements of
158        // being `u16` that come from `rowan::SyntaxKind`.
159        let mut err = "";
160
161        let syntax_kind = {
162            match kind {
163                squawk_lexer::TokenKind::LineComment => SyntaxKind::COMMENT,
164                squawk_lexer::TokenKind::BlockComment { terminated } => {
165                    if !terminated {
166                        err = "Missing trailing `*/` symbols to terminate the block comment";
167                    }
168                    SyntaxKind::COMMENT
169                }
170
171                squawk_lexer::TokenKind::Whitespace => SyntaxKind::WHITESPACE,
172                squawk_lexer::TokenKind::Ident => {
173                    // TODO: check for max identifier length
174                    //
175                    // see: https://www.postgresql.org/docs/16/sql-syntax-lexical.html#SQL-SYNTAX-IDENTIFIERS
176                    // The system uses no more than NAMEDATALEN-1 bytes of an
177                    // identifier; longer names can be written in commands, but
178                    // they will be truncated. By default, NAMEDATALEN is 64 so
179                    // the maximum identifier length is 63 bytes. If this limit
180                    // is problematic, it can be raised by changing the
181                    // NAMEDATALEN constant in src/include/pg_config_manual.h.
182                    // see: https://github.com/postgres/postgres/blob/e032e4c7ddd0e1f7865b246ec18944365d4f8614/src/include/pg_config_manual.h#L29
183                    SyntaxKind::from_keyword(token_text).unwrap_or(SyntaxKind::IDENT)
184                }
185                squawk_lexer::TokenKind::Literal { kind, .. } => {
186                    self.extend_literal(token_text.len(), kind);
187                    return;
188                }
189                squawk_lexer::TokenKind::Semi => SyntaxKind::SEMICOLON,
190                squawk_lexer::TokenKind::Comma => SyntaxKind::COMMA,
191                squawk_lexer::TokenKind::Dot => SyntaxKind::DOT,
192                squawk_lexer::TokenKind::OpenParen => SyntaxKind::L_PAREN,
193                squawk_lexer::TokenKind::CloseParen => SyntaxKind::R_PAREN,
194                squawk_lexer::TokenKind::OpenBracket => SyntaxKind::L_BRACK,
195                squawk_lexer::TokenKind::CloseBracket => SyntaxKind::R_BRACK,
196                squawk_lexer::TokenKind::At => SyntaxKind::AT,
197                squawk_lexer::TokenKind::Pound => SyntaxKind::POUND,
198                squawk_lexer::TokenKind::Tilde => SyntaxKind::TILDE,
199                squawk_lexer::TokenKind::Question => SyntaxKind::QUESTION,
200                squawk_lexer::TokenKind::Colon => SyntaxKind::COLON,
201                squawk_lexer::TokenKind::Eq => SyntaxKind::EQ,
202                squawk_lexer::TokenKind::Bang => SyntaxKind::BANG,
203                squawk_lexer::TokenKind::Lt => SyntaxKind::L_ANGLE,
204                squawk_lexer::TokenKind::Gt => SyntaxKind::R_ANGLE,
205                squawk_lexer::TokenKind::Minus => SyntaxKind::MINUS,
206                squawk_lexer::TokenKind::And => SyntaxKind::AMP,
207                squawk_lexer::TokenKind::Or => SyntaxKind::PIPE,
208                squawk_lexer::TokenKind::Plus => SyntaxKind::PLUS,
209                squawk_lexer::TokenKind::Star => SyntaxKind::STAR,
210                squawk_lexer::TokenKind::Slash => SyntaxKind::SLASH,
211                squawk_lexer::TokenKind::Caret => SyntaxKind::CARET,
212                squawk_lexer::TokenKind::Percent => SyntaxKind::PERCENT,
213                squawk_lexer::TokenKind::Unknown => SyntaxKind::ERROR,
214                squawk_lexer::TokenKind::UnknownPrefix => {
215                    err = "unknown literal prefix";
216                    SyntaxKind::IDENT
217                }
218                squawk_lexer::TokenKind::Eof => SyntaxKind::EOF,
219                squawk_lexer::TokenKind::Backtick => SyntaxKind::BACKTICK,
220                squawk_lexer::TokenKind::PositionalParam => SyntaxKind::POSITIONAL_PARAM,
221                squawk_lexer::TokenKind::QuotedIdent { terminated } => {
222                    if !terminated {
223                        err = "Missing trailing \" to terminate the quoted identifier"
224                    }
225                    SyntaxKind::IDENT
226                }
227            }
228        };
229
230        let err = if err.is_empty() { None } else { Some(err) };
231        self.push(syntax_kind, token_text.len(), err);
232    }
233
234    fn extend_literal(&mut self, len: usize, kind: &squawk_lexer::LiteralKind) {
235        let mut err = "";
236
237        let syntax_kind = match *kind {
238            squawk_lexer::LiteralKind::Int { empty_int, base: _ } => {
239                if empty_int {
240                    err = "Missing digits after the integer base prefix";
241                }
242                SyntaxKind::INT_NUMBER
243            }
244            squawk_lexer::LiteralKind::Float {
245                empty_exponent,
246                base: _,
247            } => {
248                if empty_exponent {
249                    err = "Missing digits after the exponent symbol";
250                }
251                SyntaxKind::FLOAT_NUMBER
252            }
253            squawk_lexer::LiteralKind::Str { terminated } => {
254                if !terminated {
255                    err = "Missing trailing `'` symbol to terminate the string literal";
256                }
257                // TODO: rust analzyer checks for un-escaped strings, we should too
258                SyntaxKind::STRING
259            }
260            squawk_lexer::LiteralKind::ByteStr { terminated } => {
261                if !terminated {
262                    err = "Missing trailing `'` symbol to terminate the hex bit string literal";
263                }
264                // TODO: rust analzyer checks for un-escaped strings, we should too
265                SyntaxKind::BYTE_STRING
266            }
267            squawk_lexer::LiteralKind::BitStr { terminated } => {
268                if !terminated {
269                    err = "Missing trailing `\'` symbol to terminate the bit string literal";
270                }
271                // TODO: rust analzyer checks for un-escaped strings, we should too
272                SyntaxKind::BIT_STRING
273            }
274            squawk_lexer::LiteralKind::DollarQuotedString { terminated } => {
275                if !terminated {
276                    // TODO: we could be fancier and say the ending string we're looking for
277                    err = "Unterminated dollar quoted string literal";
278                }
279                // TODO: rust analzyer checks for un-escaped strings, we should too
280                SyntaxKind::DOLLAR_QUOTED_STRING
281            }
282            squawk_lexer::LiteralKind::UnicodeEscStr { terminated } => {
283                if !terminated {
284                    err = "Missing trailing `'` symbol to terminate the unicode escape string literal";
285                }
286                // TODO: rust analzyer checks for un-escaped strings, we should too
287                SyntaxKind::BYTE_STRING
288            }
289            squawk_lexer::LiteralKind::EscStr { terminated } => {
290                if !terminated {
291                    err = "Missing trailing `\'` symbol to terminate the escape string literal";
292                }
293                // TODO: rust analzyer checks for un-escaped strings, we should too
294                SyntaxKind::ESC_STRING
295            }
296        };
297
298        let err = if err.is_empty() { None } else { Some(err) };
299        self.push(syntax_kind, len, err);
300    }
301}
squawk_parser/lexed_str.rs

squawk_parser/
lexed_str.rs
