powdb_query/
lexer.rs

1use crate::token::Token;
2
3/// Lowercase words recognized by the PowQL lexer as keywords or built-in
4/// literal words. CLI completion imports this list so it cannot drift from
5/// the parser surface.
6pub const POWQL_KEYWORDS: &[&str] = &[
7    "abs",
8    "add",
9    "alter",
10    "and",
11    "as",
12    "asc",
13    "auto",
14    "avg",
15    "begin",
16    "between",
17    "case",
18    "cast",
19    "ceil",
20    "column",
21    "commit",
22    "concat",
23    "conflict",
24    "count",
25    "cross",
26    "date_add",
27    "date_diff",
28    "default",
29    "delete",
30    "dense_rank",
31    "desc",
32    "distinct",
33    "drop",
34    "else",
35    "end",
36    "exists",
37    "explain",
38    "extract",
39    "false",
40    "filter",
41    "floor",
42    "group",
43    "having",
44    "in",
45    "index",
46    "inner",
47    "insert",
48    "is",
49    "join",
50    "left",
51    "length",
52    "let",
53    "like",
54    "limit",
55    "link",
56    "lower",
57    "match",
58    "materialize",
59    "materialized",
60    "max",
61    "min",
62    "multi",
63    "not",
64    "now",
65    "null",
66    "offset",
67    "on",
68    "or",
69    "order",
70    "outer",
71    "over",
72    "partition",
73    "pow",
74    "rank",
75    "refresh",
76    "required",
77    "returning",
78    "right",
79    "rollback",
80    "round",
81    "row_number",
82    "select",
83    "sqrt",
84    "substring",
85    "sum",
86    "then",
87    "transaction",
88    "trim",
89    "true",
90    "type",
91    "union",
92    "unique",
93    "update",
94    "upper",
95    "upsert",
96    "view",
97    "when",
98];
99
100/// Maximum allowed length for a string literal (16 MB).
101/// Prevents unbounded memory consumption from queries with multi-gigabyte strings.
102const MAX_STRING_LITERAL: usize = 16 * 1024 * 1024;
103
104#[derive(Debug)]
105pub struct LexError {
106    pub message: String,
107    pub position: usize,
108}
109
110impl std::fmt::Display for LexError {
111    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
112        write!(f, "at position {}: {}", self.position, self.message)
113    }
114}
115
116impl std::error::Error for LexError {}
117
118/// Tokenize a PowQL input string into a stream of tokens.
119///
120/// # Examples
121///
122/// ```
123/// use powdb_query::lexer::lex;
124/// use powdb_query::token::Token;
125///
126/// let tokens = lex("User filter .age > 30").unwrap();
127/// assert_eq!(tokens[0], Token::Ident("User".to_string()));
128/// assert_eq!(tokens[1], Token::Filter);
129/// assert_eq!(tokens[2], Token::DotIdent("age".to_string()));
130/// ```
131pub fn lex(input: &str) -> Result<Vec<Token>, LexError> {
132    let mut tokens = Vec::new();
133    let chars: Vec<char> = input.chars().collect();
134    let mut pos = 0;
135
136    while pos < chars.len() {
137        // Skip whitespace
138        if chars[pos].is_whitespace() {
139            pos += 1;
140            continue;
141        }
142
143        // Skip comments
144        if chars[pos] == '#' {
145            while pos < chars.len() && chars[pos] != '\n' {
146                pos += 1;
147            }
148            continue;
149        }
150
151        // Dot-ident: .fieldname
152        if chars[pos] == '.'
153            && pos + 1 < chars.len()
154            && (chars[pos + 1].is_alphabetic() || chars[pos + 1] == '_')
155        {
156            pos += 1; // skip dot
157            let start = pos;
158            while pos < chars.len() && (chars[pos].is_alphanumeric() || chars[pos] == '_') {
159                pos += 1;
160            }
161            let name: String = chars[start..pos].iter().collect();
162            tokens.push(Token::DotIdent(name));
163            continue;
164        }
165
166        // Param: $name
167        if chars[pos] == '$' {
168            pos += 1;
169            let start = pos;
170            while pos < chars.len() && (chars[pos].is_alphanumeric() || chars[pos] == '_') {
171                pos += 1;
172            }
173            let name: String = chars[start..pos].iter().collect();
174            tokens.push(Token::Param(name));
175            continue;
176        }
177
178        // String literal
179        if chars[pos] == '"' {
180            pos += 1;
181            let mut s = String::new();
182            while pos < chars.len() && chars[pos] != '"' {
183                if chars[pos] == '\\' && pos + 1 < chars.len() {
184                    match chars[pos + 1] {
185                        '"' => {
186                            s.push('"');
187                            pos += 2;
188                        }
189                        '\\' => {
190                            s.push('\\');
191                            pos += 2;
192                        }
193                        'n' => {
194                            s.push('\n');
195                            pos += 2;
196                        }
197                        't' => {
198                            s.push('\t');
199                            pos += 2;
200                        }
201                        _ => {
202                            s.push(chars[pos + 1]);
203                            pos += 2;
204                        }
205                    }
206                } else {
207                    s.push(chars[pos]);
208                    pos += 1;
209                }
210            }
211            if pos >= chars.len() {
212                return Err(LexError {
213                    message: "unterminated string".into(),
214                    position: pos,
215                });
216            }
217            pos += 1; // closing quote
218            if s.len() > MAX_STRING_LITERAL {
219                return Err(LexError {
220                    message: format!(
221                        "string literal exceeds maximum size of {}MB",
222                        MAX_STRING_LITERAL / (1024 * 1024)
223                    ),
224                    position: pos,
225                });
226            }
227            tokens.push(Token::StringLit(s));
228            continue;
229        }
230
231        // Number (int or float)
232        if chars[pos].is_ascii_digit()
233            || (chars[pos] == '-' && pos + 1 < chars.len() && chars[pos + 1].is_ascii_digit())
234        {
235            let start = pos;
236            if chars[pos] == '-' {
237                pos += 1;
238            }
239            while pos < chars.len() && chars[pos].is_ascii_digit() {
240                pos += 1;
241            }
242            if pos < chars.len()
243                && chars[pos] == '.'
244                && pos + 1 < chars.len()
245                && chars[pos + 1].is_ascii_digit()
246            {
247                pos += 1;
248                while pos < chars.len() && chars[pos].is_ascii_digit() {
249                    pos += 1;
250                }
251                let s: String = chars[start..pos].iter().collect();
252                let value = s.parse::<f64>().map_err(|_| LexError {
253                    message: format!("float literal out of range: {s}"),
254                    position: start,
255                })?;
256                tokens.push(Token::FloatLit(value));
257            } else {
258                let s: String = chars[start..pos].iter().collect();
259                let value = s.parse::<i64>().map_err(|_| LexError {
260                    message: format!("integer literal out of range for i64: {s}"),
261                    position: start,
262                })?;
263                tokens.push(Token::IntLit(value));
264            }
265            continue;
266        }
267
268        // Identifiers and keywords
269        if chars[pos].is_alphabetic() || chars[pos] == '_' {
270            let start = pos;
271            while pos < chars.len() && (chars[pos].is_alphanumeric() || chars[pos] == '_') {
272                pos += 1;
273            }
274            let word: String = chars[start..pos].iter().collect();
275            let token = match word.as_str() {
276                "type" => Token::Type,
277                "filter" => Token::Filter,
278                "order" => Token::Order,
279                "limit" => Token::Limit,
280                "offset" => Token::Offset,
281                "insert" => Token::Insert,
282                "update" => Token::Update,
283                "delete" => Token::Delete,
284                "default" => Token::Default,
285                "upsert" => Token::Upsert,
286                "returning" => Token::Returning,
287                "conflict" => Token::Conflict,
288                "select" => Token::Select,
289                "required" => Token::Required,
290                "multi" => Token::Multi,
291                "link" => Token::Link,
292                "index" => Token::Index,
293                "unique" => Token::Unique,
294                "on" => Token::On,
295                "asc" => Token::Asc,
296                "auto" => Token::Auto,
297                "desc" => Token::Desc,
298                "and" => Token::And,
299                "or" => Token::Or,
300                "not" => Token::Not,
301                "exists" => Token::Exists,
302                "let" => Token::Let,
303                "as" => Token::As,
304                "match" => Token::Match,
305                "group" => Token::Group,
306                "join" => Token::Join,
307                "inner" => Token::Inner,
308                "left" => Token::LeftKw,
309                "right" => Token::RightKw,
310                "outer" => Token::Outer,
311                "cross" => Token::Cross,
312                "transaction" => Token::Transaction,
313                "begin" => Token::Begin,
314                "commit" => Token::Commit,
315                "rollback" => Token::Rollback,
316                "view" => Token::View,
317                "materialized" => Token::Materialized,
318                "materialize" => Token::Materialized,
319                "refresh" => Token::Refresh,
320                "union" => Token::Union,
321                "having" => Token::Having,
322                "distinct" => Token::Distinct,
323                "in" => Token::In,
324                "between" => Token::Between,
325                "like" => Token::Like,
326                "count" => Token::Count,
327                "avg" => Token::Avg,
328                "sum" => Token::Sum,
329                "min" => Token::Min,
330                "max" => Token::Max,
331                "is" => Token::Is,
332                "null" => Token::Null,
333                "upper" => Token::Upper,
334                "lower" => Token::Lower,
335                "length" => Token::Length,
336                "trim" => Token::Trim,
337                "substring" => Token::Substring,
338                "concat" => Token::Concat,
339                "abs" => Token::Abs,
340                "round" => Token::Round,
341                "ceil" => Token::Ceil,
342                "floor" => Token::Floor,
343                "sqrt" => Token::Sqrt,
344                "pow" => Token::Pow,
345                "now" => Token::Now,
346                "extract" => Token::Extract,
347                "date_add" => Token::DateAdd,
348                "date_diff" => Token::DateDiff,
349                "cast" => Token::Cast,
350                "case" => Token::Case,
351                "when" => Token::When,
352                "then" => Token::Then,
353                "else" => Token::Else,
354                "end" => Token::End,
355                "over" => Token::Over,
356                "partition" => Token::Partition,
357                "row_number" => Token::RowNumber,
358                "rank" => Token::Rank,
359                "dense_rank" => Token::DenseRank,
360                "alter" => Token::Alter,
361                "drop" => Token::Drop,
362                "add" => Token::Add,
363                "column" => Token::Column,
364                "explain" => Token::Explain,
365                "true" => Token::BoolLit(true),
366                "false" => Token::BoolLit(false),
367                _ => Token::Ident(word),
368            };
369            tokens.push(token);
370            continue;
371        }
372
373        // Two-char operators
374        if pos + 1 < chars.len() {
375            let two: String = chars[pos..pos + 2].iter().collect();
376            match two.as_str() {
377                ":=" => {
378                    tokens.push(Token::Assign);
379                    pos += 2;
380                    continue;
381                }
382                "->" => {
383                    tokens.push(Token::Arrow);
384                    pos += 2;
385                    continue;
386                }
387                "!=" => {
388                    tokens.push(Token::Neq);
389                    pos += 2;
390                    continue;
391                }
392                "<=" => {
393                    tokens.push(Token::Lte);
394                    pos += 2;
395                    continue;
396                }
397                ">=" => {
398                    tokens.push(Token::Gte);
399                    pos += 2;
400                    continue;
401                }
402                "??" => {
403                    tokens.push(Token::Coalesce);
404                    pos += 2;
405                    continue;
406                }
407                _ => {}
408            }
409        }
410
411        // Single-char operators
412        let token = match chars[pos] {
413            '=' => Token::Eq,
414            '<' => Token::Lt,
415            '>' => Token::Gt,
416            '|' => Token::Pipe,
417            '+' => Token::Plus,
418            '-' => Token::Minus,
419            '*' => Token::Star,
420            '/' => Token::Slash,
421            '{' => Token::LBrace,
422            '}' => Token::RBrace,
423            '(' => Token::LParen,
424            ')' => Token::RParen,
425            ',' => Token::Comma,
426            ':' => Token::Colon,
427            '.' => Token::Dot,
428            c => {
429                return Err(LexError {
430                    message: format!("unexpected character: {c}"),
431                    position: pos,
432                })
433            }
434        };
435        tokens.push(token);
436        pos += 1;
437    }
438
439    tokens.push(Token::Eof);
440    Ok(tokens)
441}
442
443/// Split a PowQL source string into individual statements on top-level `;`,
444/// mirroring the lexer's string- and comment-scanning rules so a `;` inside a
445/// `"..."` literal or a `#` comment is never a boundary. Segments are trimmed
446/// and empty ones dropped, so leading/trailing/doubled `;` and blank lines are
447/// harmless.
448///
449/// Infallible by design: an unterminated string leaves its remainder as the
450/// final segment, so the "unterminated string" error surfaces once, at
451/// execution time, from [`lex`] — this function never reports it.
452pub fn split_statements(input: &str) -> Vec<&str> {
453    #[derive(PartialEq)]
454    enum State {
455        Normal,
456        InString,
457        InComment,
458    }
459
460    let mut out = Vec::new();
461    let mut start = 0usize;
462    let mut state = State::Normal;
463    let mut chars = input.char_indices();
464
465    while let Some((i, c)) = chars.next() {
466        match state {
467            State::Normal => match c {
468                ';' => {
469                    let seg = input[start..i].trim();
470                    if !seg.is_empty() {
471                        out.push(seg);
472                    }
473                    start = i + 1; // `;` is one byte
474                }
475                '"' => state = State::InString,
476                '#' => state = State::InComment,
477                _ => {}
478            },
479            State::InString => match c {
480                // Mirror the lexer: a backslash consumes the next char
481                // unconditionally, so `\"` and `\;` stay inside the string.
482                '\\' => {
483                    chars.next();
484                }
485                '"' => state = State::Normal,
486                _ => {}
487            },
488            State::InComment => {
489                if c == '\n' {
490                    state = State::Normal;
491                }
492            }
493        }
494    }
495
496    let seg = input[start..].trim();
497    if !seg.is_empty() {
498        out.push(seg);
499    }
500    out
501}
502
503#[cfg(test)]
504mod tests {
505    use super::*;
506    use crate::token::Token;
507
508    #[test]
509    fn test_lex_simple_query() {
510        let tokens = lex("User filter .age > 30").unwrap();
511        assert_eq!(
512            tokens,
513            vec![
514                Token::Ident("User".into()),
515                Token::Filter,
516                Token::DotIdent("age".into()),
517                Token::Gt,
518                Token::IntLit(30),
519                Token::Eof,
520            ]
521        );
522    }
523
524    #[test]
525    fn test_lex_projection() {
526        let tokens = lex("User { name, email }").unwrap();
527        assert_eq!(
528            tokens,
529            vec![
530                Token::Ident("User".into()),
531                Token::LBrace,
532                Token::Ident("name".into()),
533                Token::Comma,
534                Token::Ident("email".into()),
535                Token::RBrace,
536                Token::Eof,
537            ]
538        );
539    }
540
541    #[test]
542    fn test_lex_insert() {
543        let tokens = lex(r#"insert User { name := "Alice", age := 30 }"#).unwrap();
544        assert_eq!(
545            tokens,
546            vec![
547                Token::Insert,
548                Token::Ident("User".into()),
549                Token::LBrace,
550                Token::Ident("name".into()),
551                Token::Assign,
552                Token::StringLit("Alice".into()),
553                Token::Comma,
554                Token::Ident("age".into()),
555                Token::Assign,
556                Token::IntLit(30),
557                Token::RBrace,
558                Token::Eof,
559            ]
560        );
561    }
562
563    #[test]
564    fn test_lex_params() {
565        let tokens = lex("User filter .age > $min_age").unwrap();
566        assert_eq!(
567            tokens,
568            vec![
569                Token::Ident("User".into()),
570                Token::Filter,
571                Token::DotIdent("age".into()),
572                Token::Gt,
573                Token::Param("min_age".into()),
574                Token::Eof,
575            ]
576        );
577    }
578
579    #[test]
580    fn test_lex_string_with_escapes() {
581        let tokens = lex(r#""hello \"world\"""#).unwrap();
582        assert_eq!(
583            tokens,
584            vec![Token::StringLit("hello \"world\"".into()), Token::Eof,]
585        );
586    }
587
588    #[test]
589    fn test_lex_aggregation() {
590        let tokens = lex("count(User)").unwrap();
591        assert_eq!(
592            tokens,
593            vec![
594                Token::Count,
595                Token::LParen,
596                Token::Ident("User".into()),
597                Token::RParen,
598                Token::Eof,
599            ]
600        );
601    }
602
603    /// Regression for issue #24: an integer literal with more digits than
604    /// i64 can hold previously reached `s.parse::<i64>().unwrap()` and
605    /// panicked. It must return a `LexError` instead.
606    #[test]
607    fn test_lex_intlit_overflow_returns_err() {
608        // 22 digits — well past i64::MAX (19 digits).
609        let err = lex("4444444441111111144444").expect_err("must error, not panic");
610        assert!(
611            err.message.contains("integer literal out of range"),
612            "unexpected message: {}",
613            err.message
614        );
615        assert_eq!(err.position, 0);
616    }
617
618    /// Same bug, reached via the exact fuzzer reproducer from the
619    /// libFuzzer artifact attached to issue #24 (base64
620    /// `YXMJCQkJCQkJCQkJCQkJNDQ0NDQ0NDQ0MTExMTExMTQ0NDQJCQkJCQk=`).
621    #[test]
622    fn test_lex_fuzz_repro_issue_24() {
623        let input = "as\t\t\t\t\t\t\t\t\t\t\t\t\t44444444411111114444\t\t\t\t\t\t";
624        let err = lex(input).expect_err("fuzz reproducer must now error, not panic");
625        assert!(err.message.contains("integer literal"));
626    }
627
628    // ── split_statements (issue #150) ──────────────────────────────────
629
630    #[test]
631    fn test_split_top_level_semicolons() {
632        assert_eq!(
633            split_statements("insert A { a := 1 }; insert B { b := 2 }"),
634            vec!["insert A { a := 1 }", "insert B { b := 2 }"]
635        );
636    }
637
638    #[test]
639    fn test_split_semicolon_in_string_not_split() {
640        // The core #150 repro: a `;` inside a string literal must not split.
641        assert_eq!(
642            split_statements(r#"insert Note { body := "hello; world" }"#),
643            vec![r#"insert Note { body := "hello; world" }"#]
644        );
645    }
646
647    #[test]
648    fn test_split_escaped_quote_then_semicolon() {
649        // `\"` keeps us inside the string, so the following `;` does not
650        // split; the string closes at the final unescaped `"`, then the
651        // top-level `;` splits.
652        let input = r#"insert A { v := "a\"; b" }; insert B { c := 1 }"#;
653        assert_eq!(
654            split_statements(input),
655            vec![r#"insert A { v := "a\"; b" }"#, "insert B { c := 1 }"]
656        );
657    }
658
659    #[test]
660    fn test_split_backslash_consumes_any_char() {
661        // `"\\"` is a single-backslash string (the `\` escapes the `\`); the
662        // string closes at the second `"`, so the trailing `;` splits.
663        let input = r#"insert A { v := "\\" }; x"#;
664        assert_eq!(
665            split_statements(input),
666            vec![r#"insert A { v := "\\" }"#, "x"]
667        );
668    }
669
670    #[test]
671    fn test_split_semicolon_in_comment_not_split() {
672        let input = "insert A { a := 1 } # trailing; comment\n; insert B { b := 2 }";
673        assert_eq!(
674            split_statements(input),
675            vec![
676                "insert A { a := 1 } # trailing; comment",
677                "insert B { b := 2 }"
678            ]
679        );
680    }
681
682    #[test]
683    fn test_split_drops_empty_segments() {
684        // Leading, doubled, and trailing `;` plus blank lines all drop.
685        assert_eq!(split_statements("; A ;; B ;\n\n"), vec!["A", "B"]);
686    }
687
688    #[test]
689    fn test_split_no_semicolon_backcompat() {
690        // Byte-identical to the old `split(';').map(trim).filter(non-empty)`
691        // single-statement behavior.
692        assert_eq!(split_statements("count(User)"), vec!["count(User)"]);
693        assert!(split_statements("   ").is_empty());
694    }
695
696    #[test]
697    fn test_split_unterminated_string_tail() {
698        // Never errors: the unterminated string becomes the final segment.
699        let input = r#"insert A { a := 1 }; insert B { b := "oops"#;
700        assert_eq!(
701            split_statements(input),
702            vec!["insert A { a := 1 }", r#"insert B { b := "oops"#]
703        );
704    }
705
706    #[test]
707    fn test_split_multiline_string_with_semicolon() {
708        let input = "insert A { body := \"line1;\nline2\" }; insert B { b := 2 }";
709        assert_eq!(
710            split_statements(input),
711            vec![
712                "insert A { body := \"line1;\nline2\" }",
713                "insert B { b := 2 }"
714            ]
715        );
716    }
717}
powdb_query/lexer.rs

powdb_query/
lexer.rs
