limbo_sqlite3_parser/dialect/
mod.rs

1//! SQLite dialect
2
3use std::borrow::Cow;
4use std::fmt::Formatter;
5use std::str;
6use uncased::UncasedStr;
7
8mod token;
9pub use token::TokenType;
10
11/// Token value (lexeme).
12///
13/// The lexeme is stored as `Cow<'static, str>`: a keyword token whose source
14/// spelling matches its canonical form exactly (e.g. `SELECT`) borrows the
15/// associated `&'static str` from [`TokenType::as_str`] at zero cost, while
16/// every other token (identifiers, literals, differently-cased keywords, ...)
17/// owns a freshly allocated `String` copied from the input.
18#[derive(Clone)]
19pub struct Token(pub usize, pub Cow<'static, str>, pub usize);
20
21pub(crate) fn sentinel(start: usize) -> Token {
22    Token(start, Cow::Borrowed(""), start)
23}
24
25impl Token {
26    /// Access token value
27    pub fn unwrap(self) -> String {
28        self.1.into_owned()
29    }
30}
31
32impl std::fmt::Debug for Token {
33    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
34        f.debug_tuple("Token").field(&self.1).finish()
35    }
36}
37
38impl TokenType {
39    pub(crate) fn to_token(self, start: usize, value: &[u8], end: usize) -> Token {
40        let lexeme = match self.as_str() {
41            // Exact match with the canonical spelling: borrow the 'static
42            // constant instead of copying the input bytes.
43            Some(canonical) if canonical.as_bytes() == value => Cow::Borrowed(canonical),
44            // Anything else (differently-cased keyword, identifier, literal, ...)
45            // doesn't correspond to any 'static constant, so it must own its text.
46            _ => Cow::Owned(from_bytes(value)),
47        };
48        Token(start, lexeme, end)
49    }
50}
51
52pub(crate) fn from_bytes(bytes: &[u8]) -> String {
53    unsafe { str::from_utf8_unchecked(bytes).to_owned() }
54}
55
56include!("generated/keywords.rs");
57pub(crate) const MAX_KEYWORD_LEN: usize = 17;
58
59/// Check if `word` is a keyword
60pub fn keyword_token(word: &[u8]) -> Option<TokenType> {
61    let s = std::str::from_utf8(word).ok()?;
62    KEYWORDS.get(UncasedStr::new(s)).cloned()
63}
64
65pub(crate) fn is_identifier(name: &str) -> bool {
66    if name.is_empty() {
67        return false;
68    }
69    let bytes = name.as_bytes();
70    is_identifier_start(bytes[0])
71        && (bytes.len() == 1 || bytes[1..].iter().all(|b| is_identifier_continue(*b)))
72}
73
74pub(crate) fn is_identifier_start(b: u8) -> bool {
75    b.is_ascii_uppercase() || b == b'_' || b.is_ascii_lowercase() || b > b'\x7F'
76}
77
78pub(crate) fn is_identifier_continue(b: u8) -> bool {
79    b == b'$'
80        || b.is_ascii_digit()
81        || b.is_ascii_uppercase()
82        || b == b'_'
83        || b.is_ascii_lowercase()
84        || b > b'\x7F'
85}
86
87// keyword may become an identifier
88// see %fallback in parse.y
89pub(crate) fn from_token(_ty: u16, value: Token) -> String {
90    value.1.into_owned()
91}
92
93impl TokenType {
94    /// Return the associated string (mainly for testing)
95    pub const fn as_str(&self) -> Option<&'static str> {
96        use TokenType::*;
97        match self {
98            TK_ABORT => Some("ABORT"),
99            TK_ACTION => Some("ACTION"),
100            TK_ADD => Some("ADD"),
101            TK_AFTER => Some("AFTER"),
102            TK_ALL => Some("ALL"),
103            TK_ALTER => Some("ALTER"),
104            TK_ANALYZE => Some("ANALYZE"),
105            TK_ALWAYS => Some("ALWAYS"),
106            TK_AND => Some("AND"),
107            TK_AS => Some("AS"),
108            TK_ASC => Some("ASC"),
109            TK_ATTACH => Some("ATTACH"),
110            TK_AUTOINCR => Some("AUTOINCREMENT"),
111            TK_BEFORE => Some("BEFORE"),
112            TK_BEGIN => Some("BEGIN"),
113            TK_BETWEEN => Some("BETWEEN"),
114            TK_BY => Some("BY"),
115            TK_CASCADE => Some("CASCADE"),
116            TK_CASE => Some("CASE"),
117            TK_CAST => Some("CAST"),
118            TK_CHECK => Some("CHECK"),
119            TK_COLLATE => Some("COLLATE"),
120            TK_COLUMNKW => Some("COLUMN"),
121            TK_COMMIT => Some("COMMIT"),
122            TK_CONFLICT => Some("CONFLICT"),
123            TK_CONSTRAINT => Some("CONSTRAINT"),
124            TK_CREATE => Some("CREATE"),
125            TK_CURRENT => Some("CURRENT"),
126            TK_DATABASE => Some("DATABASE"),
127            TK_DEFAULT => Some("DEFAULT"),
128            TK_DEFERRABLE => Some("DEFERRABLE"),
129            TK_DEFERRED => Some("DEFERRED"),
130            TK_DELETE => Some("DELETE"),
131            TK_DESC => Some("DESC"),
132            TK_DETACH => Some("DETACH"),
133            TK_DISTINCT => Some("DISTINCT"),
134            TK_DO => Some("DO"),
135            TK_DROP => Some("DROP"),
136            TK_EACH => Some("EACH"),
137            TK_ELSE => Some("ELSE"),
138            TK_END => Some("END"),
139            TK_ESCAPE => Some("ESCAPE"),
140            TK_EXCEPT => Some("EXCEPT"),
141            TK_EXCLUDE => Some("EXCLUDE"),
142            TK_EXCLUSIVE => Some("EXCLUSIVE"),
143            TK_EXISTS => Some("EXISTS"),
144            TK_EXPLAIN => Some("EXPLAIN"),
145            TK_FAIL => Some("FAIL"),
146            TK_FILTER => Some("FILTER"),
147            TK_FIRST => Some("FIRST"),
148            TK_FOLLOWING => Some("FOLLOWING"),
149            TK_FOR => Some("FOR"),
150            TK_FOREIGN => Some("FOREIGN"),
151            TK_FROM => Some("FROM"),
152            TK_GENERATED => Some("GENERATED"),
153            TK_GROUP => Some("GROUP"),
154            TK_GROUPS => Some("GROUPS"),
155            TK_HAVING => Some("HAVING"),
156            TK_IF => Some("IF"),
157            TK_IGNORE => Some("IGNORE"),
158            TK_IMMEDIATE => Some("IMMEDIATE"),
159            TK_IN => Some("IN"),
160            TK_INDEX => Some("INDEX"),
161            TK_INDEXED => Some("INDEXED"),
162            TK_INITIALLY => Some("INITIALLY"),
163            TK_INSERT => Some("INSERT"),
164            TK_INSTEAD => Some("INSTEAD"),
165            TK_INTERSECT => Some("INTERSECT"),
166            TK_INTO => Some("INTO"),
167            TK_IS => Some("IS"),
168            TK_ISNULL => Some("ISNULL"),
169            TK_JOIN => Some("JOIN"),
170            TK_KEY => Some("KEY"),
171            TK_LAST => Some("LAST"),
172            TK_LIMIT => Some("LIMIT"),
173            TK_MATCH => Some("MATCH"),
174            TK_MATERIALIZED => Some("MATERIALIZED"),
175            TK_NO => Some("NO"),
176            TK_NOT => Some("NOT"),
177            TK_NOTHING => Some("NOTHING"),
178            TK_NOTNULL => Some("NOTNULL"),
179            TK_NULL => Some("NULL"),
180            TK_NULLS => Some("NULLS"),
181            TK_OF => Some("OF"),
182            TK_OFFSET => Some("OFFSET"),
183            TK_ON => Some("ON"),
184            TK_OR => Some("OR"),
185            TK_ORDER => Some("ORDER"),
186            TK_OTHERS => Some("OTHERS"),
187            TK_OVER => Some("OVER"),
188            TK_PARTITION => Some("PARTITION"),
189            TK_PLAN => Some("PLAN"),
190            TK_PRAGMA => Some("PRAGMA"),
191            TK_PRECEDING => Some("PRECEDING"),
192            TK_PRIMARY => Some("PRIMARY"),
193            TK_QUERY => Some("QUERY"),
194            TK_RAISE => Some("RAISE"),
195            TK_RANGE => Some("RANGE"),
196            TK_RECURSIVE => Some("RECURSIVE"),
197            TK_REFERENCES => Some("REFERENCES"),
198            TK_REINDEX => Some("REINDEX"),
199            TK_RELEASE => Some("RELEASE"),
200            TK_RENAME => Some("RENAME"),
201            TK_REPLACE => Some("REPLACE"),
202            TK_RETURNING => Some("RETURNING"),
203            TK_RESTRICT => Some("RESTRICT"),
204            TK_ROLLBACK => Some("ROLLBACK"),
205            TK_ROW => Some("ROW"),
206            TK_ROWS => Some("ROWS"),
207            TK_SAVEPOINT => Some("SAVEPOINT"),
208            TK_SELECT => Some("SELECT"),
209            TK_SET => Some("SET"),
210            TK_TABLE => Some("TABLE"),
211            TK_TEMP => Some("TEMP"), // or TEMPORARY
212            TK_TIES => Some("TIES"),
213            TK_THEN => Some("THEN"),
214            TK_TO => Some("TO"),
215            TK_TRANSACTION => Some("TRANSACTION"),
216            TK_TRIGGER => Some("TRIGGER"),
217            TK_UNBOUNDED => Some("UNBOUNDED"),
218            TK_UNION => Some("UNION"),
219            TK_UNIQUE => Some("UNIQUE"),
220            TK_UPDATE => Some("UPDATE"),
221            TK_USING => Some("USING"),
222            TK_VACUUM => Some("VACUUM"),
223            TK_VALUES => Some("VALUES"),
224            TK_VIEW => Some("VIEW"),
225            TK_VIRTUAL => Some("VIRTUAL"),
226            TK_WHEN => Some("WHEN"),
227            TK_WHERE => Some("WHERE"),
228            TK_WINDOW => Some("WINDOW"),
229            TK_WITH => Some("WITH"),
230            TK_WITHOUT => Some("WITHOUT"),
231            TK_BITAND => Some("&"),
232            TK_BITNOT => Some("~"),
233            TK_BITOR => Some("|"),
234            TK_COMMA => Some(","),
235            TK_CONCAT => Some("||"),
236            TK_DOT => Some("."),
237            TK_EQ => Some("="), // or ==
238            TK_GT => Some(">"),
239            TK_GE => Some(">="),
240            TK_LP => Some("("),
241            TK_LSHIFT => Some("<<"),
242            TK_LE => Some("<="),
243            TK_LT => Some("<"),
244            TK_MINUS => Some("-"),
245            TK_NE => Some("<>"), // or !=
246            TK_PLUS => Some("+"),
247            TK_REM => Some("%"),
248            TK_RP => Some(")"),
249            TK_RSHIFT => Some(">>"),
250            TK_SEMI => Some(";"),
251            TK_SLASH => Some("/"),
252            TK_STAR => Some("*"),
253            _ => None,
254        }
255    }
256}
limbo_sqlite3_parser/dialect/mod.rs

limbo_sqlite3_parser/dialect/
mod.rs
