libsql_sqlite3_parser/dialect/
mod.rs

1//! SQLite dialect
2
3use std::fmt::Formatter;
4use std::str;
5use uncased::UncasedStr;
6
7mod token;
8pub use token::TokenType;
9
10/// Token value (lexeme)
11pub struct Token(pub usize, pub Option<String>, pub usize);
12
13pub(crate) fn sentinel(start: usize) -> Token {
14    Token(start, None, start)
15}
16
17impl Token {
18    pub fn unwrap(self) -> String {
19        self.1.unwrap()
20    }
21    pub fn take(&mut self) -> Self {
22        Token(self.0, self.1.take(), self.2)
23    }
24}
25
26impl std::fmt::Debug for Token {
27    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
28        f.debug_tuple("Token").field(&self.1).finish()
29    }
30}
31
32impl TokenType {
33    // TODO try Cow<&'static, str> (Borrowed<&'static str> for keyword and Owned<String> for below),
34    // => Syntax error on keyword will be better
35    // => `from_token` will become unnecessary
36    pub(crate) fn to_token(self, start: usize, value: &[u8], end: usize) -> Token {
37        Token(
38            start,
39            match self {
40                TokenType::TK_CTIME_KW => Some(from_bytes(value)),
41                TokenType::TK_JOIN_KW => Some(from_bytes(value)),
42                TokenType::TK_LIKE_KW => Some(from_bytes(value)),
43                TokenType::TK_PTR => Some(from_bytes(value)),
44                // Identifiers
45                TokenType::TK_STRING => Some(from_bytes(value)),
46                TokenType::TK_ID => Some(from_bytes(value)),
47                TokenType::TK_VARIABLE => Some(from_bytes(value)),
48                // Values
49                TokenType::TK_ANY => Some(from_bytes(value)),
50                TokenType::TK_BLOB => Some(from_bytes(value)),
51                TokenType::TK_INTEGER => Some(from_bytes(value)),
52                TokenType::TK_FLOAT => Some(from_bytes(value)),
53                _ => None,
54            },
55            end,
56        )
57    }
58}
59
60fn from_bytes(bytes: &[u8]) -> String {
61    unsafe { str::from_utf8_unchecked(bytes).to_owned() }
62}
63
64include!(concat!(env!("OUT_DIR"), "/keywords.rs"));
65pub(crate) const MAX_KEYWORD_LEN: usize = 17;
66
67pub fn keyword_token(word: &[u8]) -> Option<TokenType> {
68    KEYWORDS
69        .get(UncasedStr::new(unsafe { str::from_utf8_unchecked(word) }))
70        .cloned()
71}
72
73pub(crate) fn is_identifier(name: &str) -> bool {
74    if name.is_empty() {
75        return false;
76    }
77    let bytes = name.as_bytes();
78    is_identifier_start(bytes[0])
79        && (bytes.len() == 1 || bytes[1..].iter().all(|b| is_identifier_continue(*b)))
80}
81
82pub(crate) fn is_identifier_start(b: u8) -> bool {
83    b.is_ascii_uppercase() || b == b'_' || b.is_ascii_lowercase() || b > b'\x7F'
84}
85
86pub(crate) fn is_identifier_continue(b: u8) -> bool {
87    b == b'$'
88        || b.is_ascii_digit()
89        || b.is_ascii_uppercase()
90        || b == b'_'
91        || b.is_ascii_lowercase()
92        || b > b'\x7F'
93}
94
95// keyword may become an identifier
96// see %fallback in parse.y
97pub(crate) fn from_token(ty: u16, value: Token) -> String {
98    use TokenType::*;
99    if let Some(str) = value.1 {
100        return str;
101    }
102    match ty {
103        x if x == TK_ABORT as u16 => "ABORT".to_owned(),
104        x if x == TK_ACTION as u16 => "ACTION".to_owned(),
105        //x if x == TK_ADD as u16 => "ADD".to_owned(),
106        x if x == TK_AFTER as u16 => "AFTER".to_owned(),
107        //x if x == TK_ALL as u16 => "ALL".to_owned(),
108        //x if x == TK_ALTER as u16 => "ALTER".to_owned(),
109        x if x == TK_ALWAYS as u16 => "ALWAYS".to_owned(),
110        x if x == TK_ANALYZE as u16 => "ANALYZE".to_owned(),
111        //x if x == TK_AND as u16 => "AND".to_owned(),
112        //x if x == TK_AS as u16 => "AS".to_owned(),
113        x if x == TK_ASC as u16 => "ASC".to_owned(),
114        x if x == TK_ATTACH as u16 => "ATTACH".to_owned(),
115        //x if x == TK_AUTOINCR as u16 => "AUTOINCREMENT".to_owned(),
116        x if x == TK_BEFORE as u16 => "BEFORE".to_owned(),
117        x if x == TK_BEGIN as u16 => "BEGIN".to_owned(),
118        //x if x == TK_BETWEEN as u16 => "BETWEEN".to_owned(),
119        x if x == TK_BY as u16 => "BY".to_owned(),
120        x if x == TK_CASCADE as u16 => "CASCADE".to_owned(),
121        //x if x == TK_CASE as u16 => "CASE".to_owned(),
122        x if x == TK_CAST as u16 => "CAST".to_owned(),
123        //x if x == TK_CHECK as u16 => "CHECK".to_owned(),
124        //x if x == TK_COLLATE as u16 => "COLLATE".to_owned(),
125        x if x == TK_COLUMNKW as u16 => "COLUMN".to_owned(),
126        //x if x == TK_COMMIT as u16 => "COMMIT".to_owned(),
127        x if x == TK_CONFLICT as u16 => "CONFLICT".to_owned(),
128        //x if x == TK_CONSTRAINT as u16 => "CONSTRAINT".to_owned(),
129        //x if x == TK_CREATE as u16 => "CREATE".to_owned(),
130        x if x == TK_CURRENT as u16 => "CURRENT".to_owned(),
131        x if x == TK_DATABASE as u16 => "DATABASE".to_owned(),
132        x if x == TK_DEFAULT as u16 => "DEFAULT".to_owned(),
133        //x if x == TK_DEFERRABLE as u16 => "DEFERRABLE".to_owned(),
134        x if x == TK_DEFERRED as u16 => "DEFERRED".to_owned(),
135        x if x == TK_DELETE as u16 => "DELETE".to_owned(),
136        x if x == TK_DESC as u16 => "DESC".to_owned(),
137        x if x == TK_DETACH as u16 => "DETACH".to_owned(),
138        //x if x == TK_DISTINCT as u16 => "DISTINCT".to_owned(),
139        x if x == TK_DO as u16 => "DO".to_owned(),
140        //x if x == TK_DROP as u16 => "DROP".to_owned(),
141        x if x == TK_EACH as u16 => "EACH".to_owned(),
142        //x if x == TK_ELSE as u16 => "ELSE".to_owned(),
143        x if x == TK_END as u16 => "END".to_owned(),
144        //x if x == TK_ESCAPE as u16 => "ESCAPE".to_owned(),
145        //x if x == TK_EXCEPT as u16 => "EXCEPT".to_owned(),
146        x if x == TK_EXCLUDE as u16 => "EXCLUDE".to_owned(),
147        x if x == TK_EXCLUSIVE as u16 => "EXCLUSIVE".to_owned(),
148        //x if x == TK_EXISTS as u16 => "EXISTS".to_owned(),
149        x if x == TK_EXPLAIN as u16 => "EXPLAIN".to_owned(),
150        x if x == TK_FAIL as u16 => "FAIL".to_owned(),
151        //x if x == TK_FILTER as u16 => "FILTER".to_owned(),
152        x if x == TK_FIRST as u16 => "FIRST".to_owned(),
153        x if x == TK_FOLLOWING as u16 => "FOLLOWING".to_owned(),
154        x if x == TK_FOR as u16 => "FOR".to_owned(),
155        //x if x == TK_FOREIGN as u16 => "FOREIGN".to_owned(),
156        //x if x == TK_FROM as u16 => "FROM".to_owned(),
157        x if x == TK_GENERATED as u16 => "GENERATED".to_owned(),
158        //x if x == TK_GROUP as u16 => "GROUP".to_owned(),
159        x if x == TK_GROUPS as u16 => "GROUPS".to_owned(),
160        //x if x == TK_HAVING as u16 => "HAVING".to_owned(),
161        x if x == TK_IF as u16 => "IF".to_owned(),
162        x if x == TK_IGNORE as u16 => "IGNORE".to_owned(),
163        x if x == TK_IMMEDIATE as u16 => "IMMEDIATE".to_owned(),
164        //x if x == TK_IN as u16 => "IN".to_owned(),
165        //x if x == TK_INDEX as u16 => "INDEX".to_owned(),
166        x if x == TK_INDEXED as u16 => "INDEXED".to_owned(),
167        x if x == TK_INITIALLY as u16 => "INITIALLY".to_owned(),
168        //x if x == TK_INSERT as u16 => "INSERT".to_owned(),
169        x if x == TK_INSTEAD as u16 => "INSTEAD".to_owned(),
170        //x if x == TK_INTERSECT as u16 => "INTERSECT".to_owned(),
171        //x if x == TK_INTO as u16 => "INTO".to_owned(),
172        //x if x == TK_IS as u16 => "IS".to_owned(),
173        //x if x == TK_ISNULL as u16 => "ISNULL".to_owned(),
174        //x if x == TK_JOIN as u16 => "JOIN".to_owned(),
175        x if x == TK_KEY as u16 => "KEY".to_owned(),
176        x if x == TK_LAST as u16 => "LAST".to_owned(),
177        //x if x == TK_LIMIT as u16 => "LIMIT".to_owned(),
178        x if x == TK_MATCH as u16 => "MATCH".to_owned(),
179        x if x == TK_MATERIALIZED as u16 => "MATERIALIZED".to_owned(),
180        x if x == TK_NO as u16 => "NO".to_owned(),
181        //x if x == TK_NOT as u16 => "NOT".to_owned(),
182        //x if x == TK_NOTHING as u16 => "NOTHING".to_owned(),
183        //x if x == TK_NOTNULL as u16 => "NOTNULL".to_owned(),
184        //x if x == TK_NULL as u16 => "NULL".to_owned(),
185        x if x == TK_NULLS as u16 => "NULLS".to_owned(),
186        x if x == TK_OF as u16 => "OF".to_owned(),
187        x if x == TK_OFFSET as u16 => "OFFSET".to_owned(),
188        x if x == TK_ON as u16 => "ON".to_owned(),
189        //x if x == TK_OR as u16 => "OR".to_owned(),
190        //x if x == TK_ORDER as u16 => "ORDER".to_owned(),
191        x if x == TK_OTHERS as u16 => "OTHERS".to_owned(),
192        //x if x == TK_OVER as u16 => "OVER".to_owned(),
193        x if x == TK_PARTITION as u16 => "PARTITION".to_owned(),
194        x if x == TK_PLAN as u16 => "PLAN".to_owned(),
195        x if x == TK_PRAGMA as u16 => "PRAGMA".to_owned(),
196        x if x == TK_PRECEDING as u16 => "PRECEDING".to_owned(),
197        //x if x == TK_PRIMARY as u16 => "PRIMARY".to_owned(),
198        x if x == TK_QUERY as u16 => "QUERY".to_owned(),
199        x if x == TK_RAISE as u16 => "RAISE".to_owned(),
200        x if x == TK_RANGE as u16 => "RANGE".to_owned(),
201        x if x == TK_READONLY as u16 => "READONLY".to_owned(),
202        x if x == TK_RECURSIVE as u16 => "RECURSIVE".to_owned(),
203        //x if x == TK_REFERENCES as u16 => "REFERENCES".to_owned(),
204        x if x == TK_REINDEX as u16 => "REINDEX".to_owned(),
205        x if x == TK_RELEASE as u16 => "RELEASE".to_owned(),
206        x if x == TK_RENAME as u16 => "RENAME".to_owned(),
207        x if x == TK_REPLACE as u16 => "REPLACE".to_owned(),
208        //x if x == TK_RETURNING as u16 => "RETURNING".to_owned(),
209        x if x == TK_RESTRICT as u16 => "RESTRICT".to_owned(),
210        x if x == TK_ROLLBACK as u16 => "ROLLBACK".to_owned(),
211        x if x == TK_ROW as u16 => "ROW".to_owned(),
212        x if x == TK_ROWS as u16 => "ROWS".to_owned(),
213        x if x == TK_SAVEPOINT as u16 => "SAVEPOINT".to_owned(),
214        //x if x == TK_SELECT as u16 => "SELECT".to_owned(),
215        //x if x == TK_SET as u16 => "SET".to_owned(),
216        //x if x == TK_TABLE as u16 => "TABLE".to_owned(),
217        x if x == TK_TEMP as u16 => "TEMP".to_owned(),
218        //x if x == TK_TEMP as u16 => "TEMPORARY".to_owned(),
219        //x if x == TK_THEN as u16 => "THEN".to_owned(),
220        x if x == TK_TIES as u16 => "TIES".to_owned(),
221        //x if x == TK_TO as u16 => "TO".to_owned(),
222        //x if x == TK_TRANSACTION as u16 => "TRANSACTION".to_owned(),
223        x if x == TK_TRIGGER as u16 => "TRIGGER".to_owned(),
224        x if x == TK_UNBOUNDED as u16 => "UNBOUNDED".to_owned(),
225        //x if x == TK_UNION as u16 => "UNION".to_owned(),
226        //x if x == TK_UNIQUE as u16 => "UNIQUE".to_owned(),
227        //x if x == TK_UPDATE as u16 => "UPDATE".to_owned(),
228        //x if x == TK_USING as u16 => "USING".to_owned(),
229        x if x == TK_VACUUM as u16 => "VACUUM".to_owned(),
230        x if x == TK_VALUES as u16 => "VALUES".to_owned(),
231        x if x == TK_VIEW as u16 => "VIEW".to_owned(),
232        x if x == TK_VIRTUAL as u16 => "VIRTUAL".to_owned(),
233        //x if x == TK_WHEN as u16 => "WHEN".to_owned(),
234        //x if x == TK_WHERE as u16 => "WHERE".to_owned(),
235        //x if x == TK_WINDOW as u16 => "WINDOW".to_owned(),
236        x if x == TK_WITH as u16 => "WITH".to_owned(),
237        x if x == TK_WITHOUT as u16 => "WITHOUT".to_owned(),
238        _ => unreachable!(),
239    }
240}
241
242impl TokenType {
243    pub const fn as_str(&self) -> Option<&'static str> {
244        use TokenType::*;
245        match self {
246            TK_ABORT => Some("ABORT"),
247            TK_ACTION => Some("ACTION"),
248            TK_ADD => Some("ADD"),
249            TK_AFTER => Some("AFTER"),
250            TK_ALL => Some("ALL"),
251            TK_ALTER => Some("ALTER"),
252            TK_ANALYZE => Some("ANALYZE"),
253            TK_ALWAYS => Some("ALWAYS"),
254            TK_AND => Some("AND"),
255            TK_AS => Some("AS"),
256            TK_ASC => Some("ASC"),
257            TK_ATTACH => Some("ATTACH"),
258            TK_AUTOINCR => Some("AUTOINCREMENT"),
259            TK_BEFORE => Some("BEFORE"),
260            TK_BEGIN => Some("BEGIN"),
261            TK_BETWEEN => Some("BETWEEN"),
262            TK_BY => Some("BY"),
263            TK_CASCADE => Some("CASCADE"),
264            TK_CASE => Some("CASE"),
265            TK_CAST => Some("CAST"),
266            TK_CHECK => Some("CHECK"),
267            TK_COLLATE => Some("COLLATE"),
268            TK_COLUMNKW => Some("COLUMN"),
269            TK_COMMIT => Some("COMMIT"),
270            TK_CONFLICT => Some("CONFLICT"),
271            TK_CONSTRAINT => Some("CONSTRAINT"),
272            TK_CREATE => Some("CREATE"),
273            TK_CURRENT => Some("CURRENT"),
274            TK_DATABASE => Some("DATABASE"),
275            TK_DEFAULT => Some("DEFAULT"),
276            TK_DEFERRABLE => Some("DEFERRABLE"),
277            TK_DEFERRED => Some("DEFERRED"),
278            TK_DELETE => Some("DELETE"),
279            TK_DESC => Some("DESC"),
280            TK_DETACH => Some("DETACH"),
281            TK_DISTINCT => Some("DISTINCT"),
282            TK_DO => Some("DO"),
283            TK_DROP => Some("DROP"),
284            TK_EACH => Some("EACH"),
285            TK_ELSE => Some("ELSE"),
286            TK_END => Some("END"),
287            TK_ESCAPE => Some("ESCAPE"),
288            TK_EXCEPT => Some("EXCEPT"),
289            TK_EXCLUDE => Some("EXCLUDE"),
290            TK_EXCLUSIVE => Some("EXCLUSIVE"),
291            TK_EXISTS => Some("EXISTS"),
292            TK_EXPLAIN => Some("EXPLAIN"),
293            TK_FAIL => Some("FAIL"),
294            TK_FILTER => Some("FILTER"),
295            TK_FIRST => Some("FIRST"),
296            TK_FOLLOWING => Some("FOLLOWING"),
297            TK_FOR => Some("FOR"),
298            TK_FOREIGN => Some("FOREIGN"),
299            TK_FROM => Some("FROM"),
300            TK_GENERATED => Some("GENERATED"),
301            TK_GROUP => Some("GROUP"),
302            TK_GROUPS => Some("GROUPS"),
303            TK_HAVING => Some("HAVING"),
304            TK_IF => Some("IF"),
305            TK_IGNORE => Some("IGNORE"),
306            TK_IMMEDIATE => Some("IMMEDIATE"),
307            TK_IN => Some("IN"),
308            TK_INDEX => Some("INDEX"),
309            TK_INDEXED => Some("INDEXED"),
310            TK_INITIALLY => Some("INITIALLY"),
311            TK_INSERT => Some("INSERT"),
312            TK_INSTEAD => Some("INSTEAD"),
313            TK_INTERSECT => Some("INTERSECT"),
314            TK_INTO => Some("INTO"),
315            TK_IS => Some("IS"),
316            TK_ISNULL => Some("ISNULL"),
317            TK_JOIN => Some("JOIN"),
318            TK_KEY => Some("KEY"),
319            TK_LAST => Some("LAST"),
320            TK_LIMIT => Some("LIMIT"),
321            TK_MATCH => Some("MATCH"),
322            TK_MATERIALIZED => Some("MATERIALIZED"),
323            TK_NO => Some("NO"),
324            TK_NOT => Some("NOT"),
325            TK_NOTHING => Some("NOTHING"),
326            TK_NOTNULL => Some("NOTNULL"),
327            TK_NULL => Some("NULL"),
328            TK_NULLS => Some("NULLS"),
329            TK_OF => Some("OF"),
330            TK_OFFSET => Some("OFFSET"),
331            TK_ON => Some("ON"),
332            TK_OR => Some("OR"),
333            TK_ORDER => Some("ORDER"),
334            TK_OTHERS => Some("OTHERS"),
335            TK_OVER => Some("OVER"),
336            TK_PARTITION => Some("PARTITION"),
337            TK_PLAN => Some("PLAN"),
338            TK_PRAGMA => Some("PRAGMA"),
339            TK_PRECEDING => Some("PRECEDING"),
340            TK_PRIMARY => Some("PRIMARY"),
341            TK_QUERY => Some("QUERY"),
342            TK_RAISE => Some("RAISE"),
343            TK_RANGE => Some("RANGE"),
344            TK_RECURSIVE => Some("RECURSIVE"),
345            TK_REFERENCES => Some("REFERENCES"),
346            TK_REINDEX => Some("REINDEX"),
347            TK_RELEASE => Some("RELEASE"),
348            TK_RENAME => Some("RENAME"),
349            TK_REPLACE => Some("REPLACE"),
350            TK_RETURNING => Some("RETURNING"),
351            TK_RESTRICT => Some("RESTRICT"),
352            TK_ROLLBACK => Some("ROLLBACK"),
353            TK_ROW => Some("ROW"),
354            TK_ROWS => Some("ROWS"),
355            TK_SAVEPOINT => Some("SAVEPOINT"),
356            TK_SELECT => Some("SELECT"),
357            TK_SET => Some("SET"),
358            TK_TABLE => Some("TABLE"),
359            TK_TEMP => Some("TEMP"), // or TEMPORARY
360            TK_TIES => Some("TIES"),
361            TK_THEN => Some("THEN"),
362            TK_TO => Some("TO"),
363            TK_TRANSACTION => Some("TRANSACTION"),
364            TK_TRIGGER => Some("TRIGGER"),
365            TK_UNBOUNDED => Some("UNBOUNDED"),
366            TK_UNION => Some("UNION"),
367            TK_UNIQUE => Some("UNIQUE"),
368            TK_UPDATE => Some("UPDATE"),
369            TK_USING => Some("USING"),
370            TK_VACUUM => Some("VACUUM"),
371            TK_VALUES => Some("VALUES"),
372            TK_VIEW => Some("VIEW"),
373            TK_VIRTUAL => Some("VIRTUAL"),
374            TK_WHEN => Some("WHEN"),
375            TK_WHERE => Some("WHERE"),
376            TK_WINDOW => Some("WINDOW"),
377            TK_WITH => Some("WITH"),
378            TK_WITHOUT => Some("WITHOUT"),
379            TK_BITAND => Some("&"),
380            TK_BITNOT => Some("~"),
381            TK_BITOR => Some("|"),
382            TK_COMMA => Some(","),
383            TK_CONCAT => Some("||"),
384            TK_DOT => Some("."),
385            TK_EQ => Some("="), // or ==
386            TK_GT => Some(">"),
387            TK_GE => Some(">="),
388            TK_LP => Some("("),
389            TK_LSHIFT => Some("<<"),
390            TK_LE => Some("<="),
391            TK_LT => Some("<"),
392            TK_MINUS => Some("-"),
393            TK_NE => Some("<>"), // or !=
394            TK_PLUS => Some("+"),
395            TK_REM => Some("%"),
396            TK_RP => Some(")"),
397            TK_RSHIFT => Some(">>"),
398            TK_SEMI => Some(";"),
399            TK_SLASH => Some("/"),
400            TK_STAR => Some("*"),
401            TK_READONLY => Some("READONLY"),
402            _ => None,
403        }
404    }
405}
libsql_sqlite3_parser/dialect/mod.rs

libsql_sqlite3_parser/dialect/
mod.rs
