powdb_query/
lexer.rs

1use crate::token::Token;
2
3/// Lowercase words recognized by the PowQL lexer as keywords or built-in
4/// literal words. CLI completion imports this list so it cannot drift from
5/// the parser surface.
6pub const POWQL_KEYWORDS: &[&str] = &[
7    "abs",
8    "add",
9    "alter",
10    "and",
11    "as",
12    "asc",
13    "auto",
14    "avg",
15    "begin",
16    "between",
17    "case",
18    "cast",
19    "ceil",
20    "column",
21    "commit",
22    "concat",
23    "conflict",
24    "count",
25    "cross",
26    "date_add",
27    "date_diff",
28    "default",
29    "delete",
30    "dense_rank",
31    "desc",
32    "distinct",
33    "drop",
34    "else",
35    "end",
36    "exists",
37    "explain",
38    "extract",
39    "false",
40    "filter",
41    "floor",
42    "group",
43    "having",
44    "in",
45    "index",
46    "inner",
47    "insert",
48    "is",
49    "join",
50    "left",
51    "length",
52    "let",
53    "like",
54    "limit",
55    "link",
56    "lower",
57    "match",
58    "materialize",
59    "materialized",
60    "max",
61    "min",
62    "multi",
63    "not",
64    "now",
65    "null",
66    "offset",
67    "on",
68    "or",
69    "order",
70    "outer",
71    "over",
72    "partition",
73    "pow",
74    "rank",
75    "refresh",
76    "required",
77    "returning",
78    "right",
79    "rollback",
80    "round",
81    "row_number",
82    "select",
83    "sqrt",
84    "substring",
85    "sum",
86    "then",
87    "transaction",
88    "trim",
89    "true",
90    "type",
91    "union",
92    "unique",
93    "update",
94    "upper",
95    "upsert",
96    "view",
97    "when",
98];
99
100/// Maximum allowed length for a string literal (16 MB).
101/// Prevents unbounded memory consumption from queries with multi-gigabyte strings.
102const MAX_STRING_LITERAL: usize = 16 * 1024 * 1024;
103
104#[derive(Debug)]
105pub struct LexError {
106    pub message: String,
107    pub position: usize,
108}
109
110impl std::fmt::Display for LexError {
111    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
112        write!(f, "at position {}: {}", self.position, self.message)
113    }
114}
115
116impl std::error::Error for LexError {}
117
118/// Tokenize a PowQL input string into a stream of tokens.
119///
120/// # Examples
121///
122/// ```
123/// use powdb_query::lexer::lex;
124/// use powdb_query::token::Token;
125///
126/// let tokens = lex("User filter .age > 30").unwrap();
127/// assert_eq!(tokens[0], Token::Ident("User".to_string()));
128/// assert_eq!(tokens[1], Token::Filter);
129/// assert_eq!(tokens[2], Token::DotIdent("age".to_string()));
130/// ```
131pub fn lex(input: &str) -> Result<Vec<Token>, LexError> {
132    let mut tokens = Vec::new();
133    let chars: Vec<char> = input.chars().collect();
134    let mut pos = 0;
135
136    while pos < chars.len() {
137        // Skip whitespace
138        if chars[pos].is_whitespace() {
139            pos += 1;
140            continue;
141        }
142
143        // Skip comments
144        if chars[pos] == '#' {
145            while pos < chars.len() && chars[pos] != '\n' {
146                pos += 1;
147            }
148            continue;
149        }
150
151        // Dot-ident: .fieldname
152        if chars[pos] == '.'
153            && pos + 1 < chars.len()
154            && (chars[pos + 1].is_alphabetic() || chars[pos + 1] == '_')
155        {
156            pos += 1; // skip dot
157            let start = pos;
158            while pos < chars.len() && (chars[pos].is_alphanumeric() || chars[pos] == '_') {
159                pos += 1;
160            }
161            let name: String = chars[start..pos].iter().collect();
162            tokens.push(Token::DotIdent(name));
163            continue;
164        }
165
166        // Param: $name
167        if chars[pos] == '$' {
168            pos += 1;
169            let start = pos;
170            while pos < chars.len() && (chars[pos].is_alphanumeric() || chars[pos] == '_') {
171                pos += 1;
172            }
173            let name: String = chars[start..pos].iter().collect();
174            tokens.push(Token::Param(name));
175            continue;
176        }
177
178        // String literal
179        if chars[pos] == '"' {
180            pos += 1;
181            let mut s = String::new();
182            while pos < chars.len() && chars[pos] != '"' {
183                if chars[pos] == '\\' && pos + 1 < chars.len() {
184                    match chars[pos + 1] {
185                        '"' => {
186                            s.push('"');
187                            pos += 2;
188                        }
189                        '\\' => {
190                            s.push('\\');
191                            pos += 2;
192                        }
193                        'n' => {
194                            s.push('\n');
195                            pos += 2;
196                        }
197                        't' => {
198                            s.push('\t');
199                            pos += 2;
200                        }
201                        _ => {
202                            s.push(chars[pos + 1]);
203                            pos += 2;
204                        }
205                    }
206                } else {
207                    s.push(chars[pos]);
208                    pos += 1;
209                }
210            }
211            if pos >= chars.len() {
212                return Err(LexError {
213                    message: "unterminated string".into(),
214                    position: pos,
215                });
216            }
217            pos += 1; // closing quote
218            if s.len() > MAX_STRING_LITERAL {
219                return Err(LexError {
220                    message: format!(
221                        "string literal exceeds maximum size of {}MB",
222                        MAX_STRING_LITERAL / (1024 * 1024)
223                    ),
224                    position: pos,
225                });
226            }
227            tokens.push(Token::StringLit(s));
228            continue;
229        }
230
231        // Number (int or float)
232        if chars[pos].is_ascii_digit()
233            || (chars[pos] == '-' && pos + 1 < chars.len() && chars[pos + 1].is_ascii_digit())
234        {
235            let start = pos;
236            if chars[pos] == '-' {
237                pos += 1;
238            }
239            while pos < chars.len() && chars[pos].is_ascii_digit() {
240                pos += 1;
241            }
242            if pos < chars.len()
243                && chars[pos] == '.'
244                && pos + 1 < chars.len()
245                && chars[pos + 1].is_ascii_digit()
246            {
247                pos += 1;
248                while pos < chars.len() && chars[pos].is_ascii_digit() {
249                    pos += 1;
250                }
251                let s: String = chars[start..pos].iter().collect();
252                let value = s.parse::<f64>().map_err(|_| LexError {
253                    message: format!("float literal out of range: {s}"),
254                    position: start,
255                })?;
256                tokens.push(Token::FloatLit(value));
257            } else {
258                let s: String = chars[start..pos].iter().collect();
259                let value = s.parse::<i64>().map_err(|_| LexError {
260                    message: format!("integer literal out of range for i64: {s}"),
261                    position: start,
262                })?;
263                tokens.push(Token::IntLit(value));
264            }
265            continue;
266        }
267
268        // Identifiers and keywords
269        if chars[pos].is_alphabetic() || chars[pos] == '_' {
270            let start = pos;
271            while pos < chars.len() && (chars[pos].is_alphanumeric() || chars[pos] == '_') {
272                pos += 1;
273            }
274            let word: String = chars[start..pos].iter().collect();
275            let token = match word.as_str() {
276                "type" => Token::Type,
277                "filter" => Token::Filter,
278                "order" => Token::Order,
279                "limit" => Token::Limit,
280                "offset" => Token::Offset,
281                "insert" => Token::Insert,
282                "update" => Token::Update,
283                "delete" => Token::Delete,
284                "default" => Token::Default,
285                "upsert" => Token::Upsert,
286                "returning" => Token::Returning,
287                "conflict" => Token::Conflict,
288                "select" => Token::Select,
289                "required" => Token::Required,
290                "multi" => Token::Multi,
291                "link" => Token::Link,
292                "index" => Token::Index,
293                "unique" => Token::Unique,
294                "on" => Token::On,
295                "asc" => Token::Asc,
296                "auto" => Token::Auto,
297                "desc" => Token::Desc,
298                "and" => Token::And,
299                "or" => Token::Or,
300                "not" => Token::Not,
301                "exists" => Token::Exists,
302                "let" => Token::Let,
303                "as" => Token::As,
304                "match" => Token::Match,
305                "group" => Token::Group,
306                "join" => Token::Join,
307                "inner" => Token::Inner,
308                "left" => Token::LeftKw,
309                "right" => Token::RightKw,
310                "outer" => Token::Outer,
311                "cross" => Token::Cross,
312                "transaction" => Token::Transaction,
313                "begin" => Token::Begin,
314                "commit" => Token::Commit,
315                "rollback" => Token::Rollback,
316                "view" => Token::View,
317                "materialized" => Token::Materialized,
318                "materialize" => Token::Materialized,
319                "refresh" => Token::Refresh,
320                "union" => Token::Union,
321                "having" => Token::Having,
322                "distinct" => Token::Distinct,
323                "in" => Token::In,
324                "between" => Token::Between,
325                "like" => Token::Like,
326                "count" => Token::Count,
327                "avg" => Token::Avg,
328                "sum" => Token::Sum,
329                "min" => Token::Min,
330                "max" => Token::Max,
331                "is" => Token::Is,
332                "null" => Token::Null,
333                "upper" => Token::Upper,
334                "lower" => Token::Lower,
335                "length" => Token::Length,
336                "trim" => Token::Trim,
337                "substring" => Token::Substring,
338                "concat" => Token::Concat,
339                "abs" => Token::Abs,
340                "round" => Token::Round,
341                "ceil" => Token::Ceil,
342                "floor" => Token::Floor,
343                "sqrt" => Token::Sqrt,
344                "pow" => Token::Pow,
345                "now" => Token::Now,
346                "extract" => Token::Extract,
347                "date_add" => Token::DateAdd,
348                "date_diff" => Token::DateDiff,
349                "cast" => Token::Cast,
350                "case" => Token::Case,
351                "when" => Token::When,
352                "then" => Token::Then,
353                "else" => Token::Else,
354                "end" => Token::End,
355                "over" => Token::Over,
356                "partition" => Token::Partition,
357                "row_number" => Token::RowNumber,
358                "rank" => Token::Rank,
359                "dense_rank" => Token::DenseRank,
360                "alter" => Token::Alter,
361                "drop" => Token::Drop,
362                "add" => Token::Add,
363                "column" => Token::Column,
364                "explain" => Token::Explain,
365                "true" => Token::BoolLit(true),
366                "false" => Token::BoolLit(false),
367                _ => Token::Ident(word),
368            };
369            tokens.push(token);
370            continue;
371        }
372
373        // Two-char operators
374        if pos + 1 < chars.len() {
375            let two: String = chars[pos..pos + 2].iter().collect();
376            match two.as_str() {
377                ":=" => {
378                    tokens.push(Token::Assign);
379                    pos += 2;
380                    continue;
381                }
382                "->" => {
383                    tokens.push(Token::Arrow);
384                    pos += 2;
385                    continue;
386                }
387                "!=" => {
388                    tokens.push(Token::Neq);
389                    pos += 2;
390                    continue;
391                }
392                "<=" => {
393                    tokens.push(Token::Lte);
394                    pos += 2;
395                    continue;
396                }
397                ">=" => {
398                    tokens.push(Token::Gte);
399                    pos += 2;
400                    continue;
401                }
402                "??" => {
403                    tokens.push(Token::Coalesce);
404                    pos += 2;
405                    continue;
406                }
407                _ => {}
408            }
409        }
410
411        // Single-char operators
412        let token = match chars[pos] {
413            '=' => Token::Eq,
414            '<' => Token::Lt,
415            '>' => Token::Gt,
416            '|' => Token::Pipe,
417            '+' => Token::Plus,
418            '-' => Token::Minus,
419            '*' => Token::Star,
420            '/' => Token::Slash,
421            '{' => Token::LBrace,
422            '}' => Token::RBrace,
423            '(' => Token::LParen,
424            ')' => Token::RParen,
425            ',' => Token::Comma,
426            ':' => Token::Colon,
427            '.' => Token::Dot,
428            c => {
429                return Err(LexError {
430                    message: format!("unexpected character: {c}"),
431                    position: pos,
432                })
433            }
434        };
435        tokens.push(token);
436        pos += 1;
437    }
438
439    tokens.push(Token::Eof);
440    Ok(tokens)
441}
442
443#[cfg(test)]
444mod tests {
445    use super::*;
446    use crate::token::Token;
447
448    #[test]
449    fn test_lex_simple_query() {
450        let tokens = lex("User filter .age > 30").unwrap();
451        assert_eq!(
452            tokens,
453            vec![
454                Token::Ident("User".into()),
455                Token::Filter,
456                Token::DotIdent("age".into()),
457                Token::Gt,
458                Token::IntLit(30),
459                Token::Eof,
460            ]
461        );
462    }
463
464    #[test]
465    fn test_lex_projection() {
466        let tokens = lex("User { name, email }").unwrap();
467        assert_eq!(
468            tokens,
469            vec![
470                Token::Ident("User".into()),
471                Token::LBrace,
472                Token::Ident("name".into()),
473                Token::Comma,
474                Token::Ident("email".into()),
475                Token::RBrace,
476                Token::Eof,
477            ]
478        );
479    }
480
481    #[test]
482    fn test_lex_insert() {
483        let tokens = lex(r#"insert User { name := "Alice", age := 30 }"#).unwrap();
484        assert_eq!(
485            tokens,
486            vec![
487                Token::Insert,
488                Token::Ident("User".into()),
489                Token::LBrace,
490                Token::Ident("name".into()),
491                Token::Assign,
492                Token::StringLit("Alice".into()),
493                Token::Comma,
494                Token::Ident("age".into()),
495                Token::Assign,
496                Token::IntLit(30),
497                Token::RBrace,
498                Token::Eof,
499            ]
500        );
501    }
502
503    #[test]
504    fn test_lex_params() {
505        let tokens = lex("User filter .age > $min_age").unwrap();
506        assert_eq!(
507            tokens,
508            vec![
509                Token::Ident("User".into()),
510                Token::Filter,
511                Token::DotIdent("age".into()),
512                Token::Gt,
513                Token::Param("min_age".into()),
514                Token::Eof,
515            ]
516        );
517    }
518
519    #[test]
520    fn test_lex_string_with_escapes() {
521        let tokens = lex(r#""hello \"world\"""#).unwrap();
522        assert_eq!(
523            tokens,
524            vec![Token::StringLit("hello \"world\"".into()), Token::Eof,]
525        );
526    }
527
528    #[test]
529    fn test_lex_aggregation() {
530        let tokens = lex("count(User)").unwrap();
531        assert_eq!(
532            tokens,
533            vec![
534                Token::Count,
535                Token::LParen,
536                Token::Ident("User".into()),
537                Token::RParen,
538                Token::Eof,
539            ]
540        );
541    }
542
543    /// Regression for issue #24: an integer literal with more digits than
544    /// i64 can hold previously reached `s.parse::<i64>().unwrap()` and
545    /// panicked. It must return a `LexError` instead.
546    #[test]
547    fn test_lex_intlit_overflow_returns_err() {
548        // 22 digits — well past i64::MAX (19 digits).
549        let err = lex("4444444441111111144444").expect_err("must error, not panic");
550        assert!(
551            err.message.contains("integer literal out of range"),
552            "unexpected message: {}",
553            err.message
554        );
555        assert_eq!(err.position, 0);
556    }
557
558    /// Same bug, reached via the exact fuzzer reproducer from the
559    /// libFuzzer artifact attached to issue #24 (base64
560    /// `YXMJCQkJCQkJCQkJCQkJNDQ0NDQ0NDQ0MTExMTExMTQ0NDQJCQkJCQk=`).
561    #[test]
562    fn test_lex_fuzz_repro_issue_24() {
563        let input = "as\t\t\t\t\t\t\t\t\t\t\t\t\t44444444411111114444\t\t\t\t\t\t";
564        let err = lex(input).expect_err("fuzz reproducer must now error, not panic");
565        assert!(err.message.contains("integer literal"));
566    }
567}
powdb_query/lexer.rs

powdb_query/
lexer.rs
