core_query/cypher/
lexer.rs

1//! Cypher subset lexer. Never panics on any `&str` input.
2
3#[derive(Debug, Clone, PartialEq)]
4pub enum Tok {
5    // keywords (case-insensitive source, canonical here)
6    Match,
7    Where,
8    Return,
9    Order,
10    By,
11    Skip,
12    Limit,
13    As,
14    And,
15    Or,
16    Not,
17    Asc,
18    Desc,
19    // pipeline keywords
20    With,
21    Unwind,
22    /// `OPTIONAL` — marks the start of an `OPTIONAL MATCH` clause.
23    Optional,
24    // write keywords
25    Create,
26    Set,
27    Delete,
28    Detach,
29    Merge,
30    Ident(String),
31    Str(String),
32    Int(i64),
33    Float(f64),
34    Param(String),
35    LParen,
36    RParen,
37    LBracket,
38    RBracket,
39    LBrace,
40    RBrace,
41    Colon,
42    /// `|` — relationship-type alternation in a pattern (`[:A|:B]`).
43    Pipe,
44    Comma,
45    Dot,
46    Eq,
47    Ne,
48    Lt,
49    Le,
50    Gt,
51    Ge,
52    Dash, // BINDING: the lexer emits `Dash`, `Lt`, `Gt` as separate tokens and the
53    // PARSER assembles rel-arrow shapes (`-[..]->`, `<-[..]-`, `-[..]-`);
54    // `<=`, `>=`, `<>` are single tokens (Le, Ge, Ne).
55    /// `*` — used in `COUNT(*)` and multiplication.
56    Star,
57    /// `+` — addition operator.
58    Plus,
59    /// `/` — division operator.
60    Slash,
61}
62
63pub fn lex(input: &str) -> Result<Vec<Tok>, String> {
64    let mut toks = Vec::new();
65    let mut chars = input.char_indices().peekable();
66    while let Some((i, ch)) = chars.next() {
67        match ch {
68            c if c.is_whitespace() => {}
69            '(' => toks.push(Tok::LParen),
70            ')' => toks.push(Tok::RParen),
71            '[' => toks.push(Tok::LBracket),
72            ']' => toks.push(Tok::RBracket),
73            '{' => toks.push(Tok::LBrace),
74            '}' => toks.push(Tok::RBrace),
75            ':' => toks.push(Tok::Colon),
76            '|' => toks.push(Tok::Pipe),
77            ',' => toks.push(Tok::Comma),
78            '.' => toks.push(Tok::Dot),
79            '=' => toks.push(Tok::Eq),
80            '*' => toks.push(Tok::Star),
81            '-' => toks.push(Tok::Dash),
82            '+' => toks.push(Tok::Plus),
83            '/' => toks.push(Tok::Slash),
84            '<' => match chars.peek() {
85                Some((_, '=')) => {
86                    chars.next();
87                    toks.push(Tok::Le);
88                }
89                Some((_, '>')) => {
90                    chars.next();
91                    toks.push(Tok::Ne);
92                }
93                _ => toks.push(Tok::Lt),
94            },
95            '>' => {
96                if matches!(chars.peek(), Some((_, '='))) {
97                    chars.next();
98                    toks.push(Tok::Ge);
99                } else {
100                    toks.push(Tok::Gt);
101                }
102            }
103            '\'' => toks.push(lex_string(i, &mut chars)?),
104            '$' => toks.push(lex_param(input, i, &mut chars)?),
105            '0'..='9' => toks.push(lex_number(input, i, ch, &mut chars)?),
106            'A'..='Z' | 'a'..='z' | '_' => toks.push(lex_word(input, i, ch, &mut chars)),
107            _ => return Err(format!("illegal character {ch:?} at position {i}")),
108        }
109    }
110    Ok(toks)
111}
112
113fn lex_string(
114    start: usize,
115    chars: &mut std::iter::Peekable<std::str::CharIndices<'_>>,
116) -> Result<Tok, String> {
117    let mut out = String::new();
118    loop {
119        match chars.next() {
120            None => return Err(format!("unterminated string at position {start}")),
121            Some((_, '\'')) => return Ok(Tok::Str(out)),
122            Some((_, '\\')) => match chars.next() {
123                Some((_, '\'')) => out.push('\''),
124                Some((pos, ch)) => {
125                    return Err(format!("invalid escape '\\{ch}' at position {pos}"));
126                }
127                None => return Err(format!("unterminated string at position {start}")),
128            },
129            Some((_, ch)) => out.push(ch),
130        }
131    }
132}
133
134fn lex_param(
135    input: &str,
136    start: usize,
137    chars: &mut std::iter::Peekable<std::str::CharIndices<'_>>,
138) -> Result<Tok, String> {
139    match chars.peek() {
140        Some((_, ch)) if is_ident_start(*ch) => Ok(Tok::Param(take_ident(input, chars))),
141        _ => Err(format!("invalid parameter at position {start}")),
142    }
143}
144
145fn lex_number(
146    input: &str,
147    start: usize,
148    first: char,
149    chars: &mut std::iter::Peekable<std::str::CharIndices<'_>>,
150) -> Result<Tok, String> {
151    let mut end = start + first.len_utf8();
152    while let Some(&(p, ch)) = chars.peek() {
153        if ch.is_ascii_digit() {
154            chars.next();
155            end = p + ch.len_utf8();
156        } else {
157            break;
158        }
159    }
160    if let Some(&(dot_pos, '.')) = chars.peek() {
161        let after_dot = input[dot_pos + '.'.len_utf8()..].chars().next();
162        if after_dot.is_some_and(|c| c.is_ascii_digit()) {
163            chars.next();
164            end = dot_pos + '.'.len_utf8();
165            while let Some(&(p, ch)) = chars.peek() {
166                if ch.is_ascii_digit() {
167                    chars.next();
168                    end = p + ch.len_utf8();
169                } else {
170                    break;
171                }
172            }
173            let val: f64 = input[start..end]
174                .parse()
175                .map_err(|_| format!("invalid float at position {start}"))?;
176            return Ok(Tok::Float(val));
177        }
178        // `..` (two consecutive dots) is the range separator for variable-length
179        // path patterns (`*1..5`).  When after_dot is also '.', stop the number
180        // here without consuming the first dot — the main lex loop will emit two
181        // `Dot` tokens for the `..` separator.
182        if after_dot == Some('.') {
183            // fall through to return the integer below
184        } else {
185            return Err(format!(
186                "invalid number at position {start}: expected digit after decimal point"
187            ));
188        }
189    }
190    let val: i64 = input[start..end]
191        .parse()
192        .map_err(|_| format!("invalid integer at position {start}"))?;
193    Ok(Tok::Int(val))
194}
195
196fn lex_word(
197    input: &str,
198    start: usize,
199    first: char,
200    chars: &mut std::iter::Peekable<std::str::CharIndices<'_>>,
201) -> Tok {
202    let mut end = start + first.len_utf8();
203    while let Some(&(p, ch)) = chars.peek() {
204        if is_ident_cont(ch) {
205            chars.next();
206            end = p + ch.len_utf8();
207        } else {
208            break;
209        }
210    }
211    keyword(&input[start..end]).unwrap_or_else(|| Tok::Ident(input[start..end].to_string()))
212}
213
214fn take_ident(input: &str, chars: &mut std::iter::Peekable<std::str::CharIndices<'_>>) -> String {
215    let (start, first) = match chars.next() {
216        Some(pair) => pair,
217        None => return String::new(),
218    };
219    let mut end = start + first.len_utf8();
220    while let Some(&(p, ch)) = chars.peek() {
221        if is_ident_cont(ch) {
222            chars.next();
223            end = p + ch.len_utf8();
224        } else {
225            break;
226        }
227    }
228    input[start..end].to_string()
229}
230
231fn is_ident_start(ch: char) -> bool {
232    ch.is_ascii_alphabetic() || ch == '_'
233}
234
235fn is_ident_cont(ch: char) -> bool {
236    ch.is_ascii_alphanumeric() || ch == '_'
237}
238
239fn keyword(word: &str) -> Option<Tok> {
240    Some(match word.to_ascii_lowercase().as_str() {
241        "match" => Tok::Match,
242        "where" => Tok::Where,
243        "return" => Tok::Return,
244        "order" => Tok::Order,
245        "by" => Tok::By,
246        "skip" => Tok::Skip,
247        "limit" => Tok::Limit,
248        "as" => Tok::As,
249        "and" => Tok::And,
250        "or" => Tok::Or,
251        "not" => Tok::Not,
252        "asc" => Tok::Asc,
253        "desc" => Tok::Desc,
254        "with" => Tok::With,
255        "unwind" => Tok::Unwind,
256        "optional" => Tok::Optional,
257        "create" => Tok::Create,
258        "set" => Tok::Set,
259        "delete" => Tok::Delete,
260        "detach" => Tok::Detach,
261        "merge" => Tok::Merge,
262        _ => return None,
263    })
264}
265
266#[cfg(test)]
267mod tests {
268    use super::{lex, Tok};
269
270    #[test]
271    fn keywords_are_case_insensitive() {
272        let expected = vec![Tok::Match];
273        assert_eq!(lex("MATCH").unwrap(), expected);
274        assert_eq!(lex("match").unwrap(), expected);
275        assert_eq!(lex("MaTcH").unwrap(), expected);
276        assert_eq!(
277            lex("WHERE RETURN ORDER BY SKIP LIMIT AS AND OR NOT ASC DESC").unwrap(),
278            vec![
279                Tok::Where,
280                Tok::Return,
281                Tok::Order,
282                Tok::By,
283                Tok::Skip,
284                Tok::Limit,
285                Tok::As,
286                Tok::And,
287                Tok::Or,
288                Tok::Not,
289                Tok::Asc,
290                Tok::Desc,
291            ]
292        );
293        assert_eq!(
294            lex("where return order by skip limit as and or not asc desc").unwrap(),
295            vec![
296                Tok::Where,
297                Tok::Return,
298                Tok::Order,
299                Tok::By,
300                Tok::Skip,
301                Tok::Limit,
302                Tok::As,
303                Tok::And,
304                Tok::Or,
305                Tok::Not,
306                Tok::Asc,
307                Tok::Desc,
308            ]
309        );
310    }
311
312    #[test]
313    fn comparison_ops_disambiguate() {
314        assert_eq!(lex("<").unwrap(), vec![Tok::Lt]);
315        assert_eq!(lex("<=").unwrap(), vec![Tok::Le]);
316        assert_eq!(lex("<>").unwrap(), vec![Tok::Ne]);
317        assert_eq!(lex(">").unwrap(), vec![Tok::Gt]);
318        assert_eq!(lex(">=").unwrap(), vec![Tok::Ge]);
319        assert_eq!(lex("=").unwrap(), vec![Tok::Eq]);
320        assert_eq!(
321            lex("< <= <> > >=").unwrap(),
322            vec![Tok::Lt, Tok::Le, Tok::Ne, Tok::Gt, Tok::Ge]
323        );
324    }
325
326    #[test]
327    fn string_escape_apostrophe() {
328        assert_eq!(lex(r"'it\'s'").unwrap(), vec![Tok::Str("it's".into())]);
329    }
330
331    #[test]
332    fn float_vs_int_vs_bare_dot_is_error() {
333        assert_eq!(lex("42").unwrap(), vec![Tok::Int(42)]);
334        assert_eq!(lex("2.5").unwrap(), vec![Tok::Float(2.5)]);
335        let err = lex("1.").expect_err("digit-dot with no following digit is an error");
336        assert!(
337            err.contains("position"),
338            "error must include position info, got: {err}"
339        );
340    }
341
342    #[test]
343    fn dollar_param() {
344        assert_eq!(lex("$name").unwrap(), vec![Tok::Param("name".into())]);
345        assert_eq!(lex("$tid").unwrap(), vec![Tok::Param("tid".into())]);
346    }
347
348    #[test]
349    fn unterminated_string_is_err_with_position() {
350        let err = lex("'abc").expect_err("unterminated string must be Err");
351        assert!(
352            err.contains("position"),
353            "error must include position info, got: {err}"
354        );
355        let err = lex("MATCH 'oops").expect_err("unterminated after tokens");
356        assert!(
357            err.contains("position"),
358            "error must include position info, got: {err}"
359        );
360    }
361
362    #[test]
363    fn composite_query_emits_every_token_variant() {
364        // Every Tok variant appears at least once. Rel arrows stay as Dash/Lt/Gt.
365        let src = r"MATCH (n:L {k: 'it\'s', i: 1, f: 2.5})-[r]->(m) WHERE NOT n.a = $p AND n.b <> 0 OR n.c < 1 AND n.d <= 2 AND n.e > 3 AND n.f >= 4.0 RETURN n, n.a AS x ORDER BY x ASC, n.b DESC SKIP 0 LIMIT 10";
366        assert_eq!(
367            lex(src).unwrap(),
368            vec![
369                Tok::Match,
370                Tok::LParen,
371                Tok::Ident("n".into()),
372                Tok::Colon,
373                Tok::Ident("L".into()),
374                Tok::LBrace,
375                Tok::Ident("k".into()),
376                Tok::Colon,
377                Tok::Str("it's".into()),
378                Tok::Comma,
379                Tok::Ident("i".into()),
380                Tok::Colon,
381                Tok::Int(1),
382                Tok::Comma,
383                Tok::Ident("f".into()),
384                Tok::Colon,
385                Tok::Float(2.5),
386                Tok::RBrace,
387                Tok::RParen,
388                Tok::Dash,
389                Tok::LBracket,
390                Tok::Ident("r".into()),
391                Tok::RBracket,
392                Tok::Dash,
393                Tok::Gt,
394                Tok::LParen,
395                Tok::Ident("m".into()),
396                Tok::RParen,
397                Tok::Where,
398                Tok::Not,
399                Tok::Ident("n".into()),
400                Tok::Dot,
401                Tok::Ident("a".into()),
402                Tok::Eq,
403                Tok::Param("p".into()),
404                Tok::And,
405                Tok::Ident("n".into()),
406                Tok::Dot,
407                Tok::Ident("b".into()),
408                Tok::Ne,
409                Tok::Int(0),
410                Tok::Or,
411                Tok::Ident("n".into()),
412                Tok::Dot,
413                Tok::Ident("c".into()),
414                Tok::Lt,
415                Tok::Int(1),
416                Tok::And,
417                Tok::Ident("n".into()),
418                Tok::Dot,
419                Tok::Ident("d".into()),
420                Tok::Le,
421                Tok::Int(2),
422                Tok::And,
423                Tok::Ident("n".into()),
424                Tok::Dot,
425                Tok::Ident("e".into()),
426                Tok::Gt,
427                Tok::Int(3),
428                Tok::And,
429                Tok::Ident("n".into()),
430                Tok::Dot,
431                Tok::Ident("f".into()),
432                Tok::Ge,
433                Tok::Float(4.0),
434                Tok::Return,
435                Tok::Ident("n".into()),
436                Tok::Comma,
437                Tok::Ident("n".into()),
438                Tok::Dot,
439                Tok::Ident("a".into()),
440                Tok::As,
441                Tok::Ident("x".into()),
442                Tok::Order,
443                Tok::By,
444                Tok::Ident("x".into()),
445                Tok::Asc,
446                Tok::Comma,
447                Tok::Ident("n".into()),
448                Tok::Dot,
449                Tok::Ident("b".into()),
450                Tok::Desc,
451                Tok::Skip,
452                Tok::Int(0),
453                Tok::Limit,
454                Tok::Int(10),
455            ]
456        );
457    }
458
459    #[test]
460    fn plus_and_slash_lex_as_tokens() {
461        assert_eq!(lex("+").unwrap(), vec![Tok::Plus]);
462        assert_eq!(lex("/").unwrap(), vec![Tok::Slash]);
463        assert_eq!(
464            lex("1 + 2 / 3").unwrap(),
465            vec![Tok::Int(1), Tok::Plus, Tok::Int(2), Tok::Slash, Tok::Int(3)]
466        );
467    }
468
469    #[test]
470    fn garbage_bytes_are_err_not_panic() {
471        let cases = ["@", "#", "MATCH @ n", "\"double\"", "\u{0}", "1.2.3@"];
472        for src in cases {
473            let result = std::panic::catch_unwind(|| lex(src));
474            assert!(result.is_ok(), "lex({src:?}) panicked");
475            let err = result
476                .unwrap()
477                .expect_err(&format!("lex({src:?}) must be Err"));
478            assert!(
479                err.contains("position"),
480                "error must include position info, got: {err}"
481            );
482        }
483    }
484}
core_query/cypher/lexer.rs

core_query/cypher/
lexer.rs
