limbo_sqlite3_parser/dialect/
mod.rs

1//! SQLite dialect
2
3use std::fmt::Formatter;
4use std::str;
5use uncased::UncasedStr;
6
7mod token;
8pub use token::TokenType;
9
10/// Token value (lexeme)
11#[derive(Clone, Copy)]
12pub struct Token<'i>(pub usize, pub &'i [u8], pub usize);
13
14pub(crate) fn sentinel(start: usize) -> Token<'static> {
15    Token(start, b"", start)
16}
17
18impl Token<'_> {
19    /// Access token value
20    pub fn unwrap(self) -> String {
21        from_bytes(self.1)
22    }
23}
24
25impl std::fmt::Debug for Token<'_> {
26    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
27        f.debug_tuple("Token").field(&self.1).finish()
28    }
29}
30
31impl TokenType {
32    // TODO try Cow<&'static, str> (Borrowed<&'static str> for keyword and Owned<String> for below),
33    // => Syntax error on keyword will be better
34    // => `from_token` will become unnecessary
35    pub(crate) fn to_token(self, start: usize, value: &[u8], end: usize) -> Token<'_> {
36        Token(start, value, end)
37    }
38}
39
40pub(crate) fn from_bytes(bytes: &[u8]) -> String {
41    unsafe { str::from_utf8_unchecked(bytes).to_owned() }
42}
43
44include!("generated/keywords.rs");
45pub(crate) const MAX_KEYWORD_LEN: usize = 17;
46
47/// Check if `word` is a keyword
48pub fn keyword_token(word: &[u8]) -> Option<TokenType> {
49    let s = std::str::from_utf8(word).ok()?;
50    KEYWORDS.get(UncasedStr::new(s)).cloned()
51}
52
53pub(crate) fn is_identifier(name: &str) -> bool {
54    if name.is_empty() {
55        return false;
56    }
57    let bytes = name.as_bytes();
58    is_identifier_start(bytes[0])
59        && (bytes.len() == 1 || bytes[1..].iter().all(|b| is_identifier_continue(*b)))
60}
61
62pub(crate) fn is_identifier_start(b: u8) -> bool {
63    b.is_ascii_uppercase() || b == b'_' || b.is_ascii_lowercase() || b > b'\x7F'
64}
65
66pub(crate) fn is_identifier_continue(b: u8) -> bool {
67    b == b'$'
68        || b.is_ascii_digit()
69        || b.is_ascii_uppercase()
70        || b == b'_'
71        || b.is_ascii_lowercase()
72        || b > b'\x7F'
73}
74
75// keyword may become an identifier
76// see %fallback in parse.y
77pub(crate) fn from_token(_ty: u16, value: Token) -> String {
78    from_bytes(value.1)
79}
80
81impl TokenType {
82    /// Return the associated string (mainly for testing)
83    pub const fn as_str(&self) -> Option<&'static str> {
84        use TokenType::*;
85        match self {
86            TK_ABORT => Some("ABORT"),
87            TK_ACTION => Some("ACTION"),
88            TK_ADD => Some("ADD"),
89            TK_AFTER => Some("AFTER"),
90            TK_ALL => Some("ALL"),
91            TK_ALTER => Some("ALTER"),
92            TK_ANALYZE => Some("ANALYZE"),
93            TK_ALWAYS => Some("ALWAYS"),
94            TK_AND => Some("AND"),
95            TK_AS => Some("AS"),
96            TK_ASC => Some("ASC"),
97            TK_ATTACH => Some("ATTACH"),
98            TK_AUTOINCR => Some("AUTOINCREMENT"),
99            TK_BEFORE => Some("BEFORE"),
100            TK_BEGIN => Some("BEGIN"),
101            TK_BETWEEN => Some("BETWEEN"),
102            TK_BY => Some("BY"),
103            TK_CASCADE => Some("CASCADE"),
104            TK_CASE => Some("CASE"),
105            TK_CAST => Some("CAST"),
106            TK_CHECK => Some("CHECK"),
107            TK_COLLATE => Some("COLLATE"),
108            TK_COLUMNKW => Some("COLUMN"),
109            TK_COMMIT => Some("COMMIT"),
110            TK_CONFLICT => Some("CONFLICT"),
111            TK_CONSTRAINT => Some("CONSTRAINT"),
112            TK_CREATE => Some("CREATE"),
113            TK_CURRENT => Some("CURRENT"),
114            TK_DATABASE => Some("DATABASE"),
115            TK_DEFAULT => Some("DEFAULT"),
116            TK_DEFERRABLE => Some("DEFERRABLE"),
117            TK_DEFERRED => Some("DEFERRED"),
118            TK_DELETE => Some("DELETE"),
119            TK_DESC => Some("DESC"),
120            TK_DETACH => Some("DETACH"),
121            TK_DISTINCT => Some("DISTINCT"),
122            TK_DO => Some("DO"),
123            TK_DROP => Some("DROP"),
124            TK_EACH => Some("EACH"),
125            TK_ELSE => Some("ELSE"),
126            TK_END => Some("END"),
127            TK_ESCAPE => Some("ESCAPE"),
128            TK_EXCEPT => Some("EXCEPT"),
129            TK_EXCLUDE => Some("EXCLUDE"),
130            TK_EXCLUSIVE => Some("EXCLUSIVE"),
131            TK_EXISTS => Some("EXISTS"),
132            TK_EXPLAIN => Some("EXPLAIN"),
133            TK_FAIL => Some("FAIL"),
134            TK_FILTER => Some("FILTER"),
135            TK_FIRST => Some("FIRST"),
136            TK_FOLLOWING => Some("FOLLOWING"),
137            TK_FOR => Some("FOR"),
138            TK_FOREIGN => Some("FOREIGN"),
139            TK_FROM => Some("FROM"),
140            TK_GENERATED => Some("GENERATED"),
141            TK_GROUP => Some("GROUP"),
142            TK_GROUPS => Some("GROUPS"),
143            TK_HAVING => Some("HAVING"),
144            TK_IF => Some("IF"),
145            TK_IGNORE => Some("IGNORE"),
146            TK_IMMEDIATE => Some("IMMEDIATE"),
147            TK_IN => Some("IN"),
148            TK_INDEX => Some("INDEX"),
149            TK_INDEXED => Some("INDEXED"),
150            TK_INITIALLY => Some("INITIALLY"),
151            TK_INSERT => Some("INSERT"),
152            TK_INSTEAD => Some("INSTEAD"),
153            TK_INTERSECT => Some("INTERSECT"),
154            TK_INTO => Some("INTO"),
155            TK_IS => Some("IS"),
156            TK_ISNULL => Some("ISNULL"),
157            TK_JOIN => Some("JOIN"),
158            TK_KEY => Some("KEY"),
159            TK_LAST => Some("LAST"),
160            TK_LIMIT => Some("LIMIT"),
161            TK_MATCH => Some("MATCH"),
162            TK_MATERIALIZED => Some("MATERIALIZED"),
163            TK_NO => Some("NO"),
164            TK_NOT => Some("NOT"),
165            TK_NOTHING => Some("NOTHING"),
166            TK_NOTNULL => Some("NOTNULL"),
167            TK_NULL => Some("NULL"),
168            TK_NULLS => Some("NULLS"),
169            TK_OF => Some("OF"),
170            TK_OFFSET => Some("OFFSET"),
171            TK_ON => Some("ON"),
172            TK_OR => Some("OR"),
173            TK_ORDER => Some("ORDER"),
174            TK_OTHERS => Some("OTHERS"),
175            TK_OVER => Some("OVER"),
176            TK_PARTITION => Some("PARTITION"),
177            TK_PLAN => Some("PLAN"),
178            TK_PRAGMA => Some("PRAGMA"),
179            TK_PRECEDING => Some("PRECEDING"),
180            TK_PRIMARY => Some("PRIMARY"),
181            TK_QUERY => Some("QUERY"),
182            TK_RAISE => Some("RAISE"),
183            TK_RANGE => Some("RANGE"),
184            TK_RECURSIVE => Some("RECURSIVE"),
185            TK_REFERENCES => Some("REFERENCES"),
186            TK_REINDEX => Some("REINDEX"),
187            TK_RELEASE => Some("RELEASE"),
188            TK_RENAME => Some("RENAME"),
189            TK_REPLACE => Some("REPLACE"),
190            TK_RETURNING => Some("RETURNING"),
191            TK_RESTRICT => Some("RESTRICT"),
192            TK_ROLLBACK => Some("ROLLBACK"),
193            TK_ROW => Some("ROW"),
194            TK_ROWS => Some("ROWS"),
195            TK_SAVEPOINT => Some("SAVEPOINT"),
196            TK_SELECT => Some("SELECT"),
197            TK_SET => Some("SET"),
198            TK_TABLE => Some("TABLE"),
199            TK_TEMP => Some("TEMP"), // or TEMPORARY
200            TK_TIES => Some("TIES"),
201            TK_THEN => Some("THEN"),
202            TK_TO => Some("TO"),
203            TK_TRANSACTION => Some("TRANSACTION"),
204            TK_TRIGGER => Some("TRIGGER"),
205            TK_UNBOUNDED => Some("UNBOUNDED"),
206            TK_UNION => Some("UNION"),
207            TK_UNIQUE => Some("UNIQUE"),
208            TK_UPDATE => Some("UPDATE"),
209            TK_USING => Some("USING"),
210            TK_VACUUM => Some("VACUUM"),
211            TK_VALUES => Some("VALUES"),
212            TK_VIEW => Some("VIEW"),
213            TK_VIRTUAL => Some("VIRTUAL"),
214            TK_WHEN => Some("WHEN"),
215            TK_WHERE => Some("WHERE"),
216            TK_WINDOW => Some("WINDOW"),
217            TK_WITH => Some("WITH"),
218            TK_WITHOUT => Some("WITHOUT"),
219            TK_BITAND => Some("&"),
220            TK_BITNOT => Some("~"),
221            TK_BITOR => Some("|"),
222            TK_COMMA => Some(","),
223            TK_CONCAT => Some("||"),
224            TK_DOT => Some("."),
225            TK_EQ => Some("="), // or ==
226            TK_GT => Some(">"),
227            TK_GE => Some(">="),
228            TK_LP => Some("("),
229            TK_LSHIFT => Some("<<"),
230            TK_LE => Some("<="),
231            TK_LT => Some("<"),
232            TK_MINUS => Some("-"),
233            TK_NE => Some("<>"), // or !=
234            TK_PLUS => Some("+"),
235            TK_REM => Some("%"),
236            TK_RP => Some(")"),
237            TK_RSHIFT => Some(">>"),
238            TK_SEMI => Some(";"),
239            TK_SLASH => Some("/"),
240            TK_STAR => Some("*"),
241            _ => None,
242        }
243    }
244}
limbo_sqlite3_parser/dialect/mod.rs

limbo_sqlite3_parser/dialect/
mod.rs
