relate-parser 1.0.1

Cypher lexer, parser, and AST for the Relate graph database
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
use super::string::decode_char;
use super::*;

fn tok_types(input: &str) -> Vec<TokenType> {
    tokenize(input).iter().map(|t| t.ty).collect()
}

fn tok_literals(input: &str) -> Vec<String> {
    tokenize(input)
        .iter()
        .map(|t| t.literal.to_string())
        .collect()
}

#[test]
fn test_decode_char_bounds_validation_to_its_sequence() {
    // PR #61 review: `decode_char` must validate only the 2–4 byte sequence
    // it decodes, not the whole remaining input. A trailing invalid byte
    // must not invalidate a leading well-formed character — otherwise
    // `skip_whitespace` re-validates the tail at every token boundary,
    // making lexing quadratic on Unicode-whitespace-heavy input.
    assert_eq!(decode_char(&[0xC2, 0xA0, 0xFF]), Some(('\u{00A0}', 2)));
    assert_eq!(
        decode_char(&[0xF0, 0x9F, 0x98, 0x80, 0xFF]),
        Some(('\u{1F600}', 4))
    );
    // A truncated sequence at end of input is still reported as malformed.
    assert_eq!(decode_char(&[0xC2]), None);
    // A multibyte non-whitespace char decodes to itself and is classified
    // as non-whitespace, so it terminates whitespace skipping.
    let (c, len) = decode_char("\u{00E9}".as_bytes()).unwrap();
    assert_eq!((c, len), ('\u{00E9}', 2));
    assert!(!c.is_whitespace());
}

#[test]
fn test_unicode_whitespace_between_tokens() {
    // NBSP (U+00A0, 2-byte) between tokens is skipped like ASCII whitespace.
    assert_eq!(
        tok_types("RETURN\u{00A0}1"),
        vec![TokenType::RETURN, TokenType::Integer]
    );
    // EM SPACE (U+2003, 3-byte) as well.
    assert_eq!(
        tok_types("MATCH\u{2003}RETURN"),
        vec![TokenType::MATCH, TokenType::RETURN]
    );
}

#[test]
fn test_worst_case_unicode_whitespace_run_tokenizes() {
    // `RETURN` followed by a long run of NBSP lexes correctly; each decode
    // validates only its own 2-byte sequence, not the remaining tail.
    let mut q = String::from("RETURN");
    q.extend(std::iter::repeat_n('\u{00A0}', 4096));
    q.push('1');
    assert_eq!(tok_types(&q), vec![TokenType::RETURN, TokenType::Integer]);
}

#[test]
fn test_multibyte_non_whitespace_is_not_skipped() {
    // U+00A7 is multibyte but not whitespace and not an identifier
    // character, so whitespace skipping must stop there rather than consume
    // it; the lexer then flags the stray byte(s) instead of silently
    // dropping them.
    let tokens = tokenize("\u{00A0}\u{00A7}");
    assert!(!tokens.is_empty());
    assert!(tokens.iter().all(|t| t.ty == TokenType::Error));
}

#[test]
fn test_invalid_multibyte_char_is_one_error_with_detail() {
    // LABS-188: a non-ASCII character outside strings/comments/whitespace
    // that is not an identifier start is ONE invalid character, not one
    // mojibake error token per UTF-8 byte, and carries the TCK
    // `InvalidUnicodeCharacter` detail. The probes stay invalid once
    // Unicode identifiers land (dashes, emoji, section sign, BOM).
    for c in [
        '\u{2014}',  // em dash
        '\u{2013}',  // en dash
        '\u{2212}',  // minus sign
        '\u{2010}',  // hyphen
        '\u{1F600}', // emoji
        '\u{00A7}',  // section sign
        '\u{FEFF}',  // byte-order mark (not whitespace)
    ] {
        let owned = c.to_string();
        let tokens = tokenize(&owned);
        assert_eq!(tokens.len(), 1, "expected one token for {c:?}: {tokens:?}");
        assert_eq!(tokens[0].ty, TokenType::Error, "{c:?}");
        assert_eq!(
            tokens[0].detail,
            Some("InvalidUnicodeCharacter"),
            "expected InvalidUnicodeCharacter for {c:?}, got {:?}",
            tokens[0].detail
        );
        assert!(
            tokens[0].literal.contains(c),
            "literal must contain the decoded {c:?}: {:?}",
            tokens[0].literal
        );
    }
}

#[test]
fn test_invalid_unicode_hyphen_lexes_following_integer() {
    // The Mathematical3 [1] query: the em dash is one error token and lexing
    // resumes after all three of its bytes, so `41` is still an Integer
    // token at its own (byte) column.
    let tokens = tokenize("RETURN 42 \u{2014} 41");
    let types: Vec<_> = tokens.iter().map(|t| t.ty).collect();
    assert_eq!(
        types,
        vec![
            TokenType::RETURN,
            TokenType::Integer,
            TokenType::Error,
            TokenType::Integer,
        ]
    );
    let err = &tokens[2];
    assert_eq!(err.detail, Some("InvalidUnicodeCharacter"));
    assert!(err.literal.contains('\u{2014}'));
    assert_eq!(err.col, 11);
    assert_eq!(tokens[3].literal, "41");
    assert_eq!(tokens[3].col, 15);
}

#[test]
fn test_ascii_fallback_keeps_unexpected_character_without_detail() {
    // ASCII characters that reach the fallback arm keep the historical
    // untagged `unexpected character:` error.
    for (query, ch) in [("RETURN 1 # 2", '#'), ("RETURN 1 ? 2", '?')] {
        let tokens = tokenize(query);
        let err = tokens
            .iter()
            .find(|t| t.ty == TokenType::Error)
            .unwrap_or_else(|| panic!("expected an error token for {query:?}"));
        assert_eq!(err.detail, None, "{query:?}");
        assert_eq!(err.literal, format!("unexpected character: {ch}"));
    }
}

#[test]
fn test_simple_punctuation() {
    assert_eq!(
        tok_types("()[]{}:,;|*+/%^="),
        vec![
            TokenType::LeftParen,
            TokenType::RightParen,
            TokenType::LeftBracket,
            TokenType::RightBracket,
            TokenType::LeftBrace,
            TokenType::RightBrace,
            TokenType::Colon,
            TokenType::Comma,
            TokenType::Semicolon,
            TokenType::Pipe,
            TokenType::Star,
            TokenType::Plus,
            TokenType::Slash,
            TokenType::Percent,
            TokenType::Caret,
            TokenType::Equals,
        ]
    );
}

#[test]
fn test_arrows_and_comparisons() {
    assert_eq!(
        tok_types("-> <- <> <= >= < > - ."),
        vec![
            TokenType::ArrowRight,
            TokenType::ArrowLeft,
            TokenType::NotEquals,
            TokenType::LessEqual,
            TokenType::GreaterEqual,
            TokenType::LessThan,
            TokenType::GreaterThan,
            TokenType::Dash,
            TokenType::Dot,
        ]
    );
}

#[test]
fn test_dot_dot() {
    assert_eq!(tok_types(".."), vec![TokenType::DotDot]);
    assert_eq!(
        tok_types("1..5"),
        vec![TokenType::Integer, TokenType::DotDot, TokenType::Integer]
    );
}

#[test]
fn test_keywords() {
    assert_eq!(
        tok_types("MATCH RETURN WHERE CREATE"),
        vec![
            TokenType::MATCH,
            TokenType::RETURN,
            TokenType::WHERE,
            TokenType::CREATE,
        ]
    );
    // Case-insensitive
    assert_eq!(
        tok_types("match return where create"),
        vec![
            TokenType::MATCH,
            TokenType::RETURN,
            TokenType::WHERE,
            TokenType::CREATE,
        ]
    );
}

#[test]
fn test_identifiers() {
    let tokens = tokenize("n foo bar123 _x");
    assert_eq!(tokens.len(), 4);
    for t in &tokens {
        assert_eq!(t.ty, TokenType::Identifier);
    }
    assert_eq!(tokens[0].literal, "n");
    assert_eq!(tokens[1].literal, "foo");
    assert_eq!(tokens[2].literal, "bar123");
    assert_eq!(tokens[3].literal, "_x");
}

#[test]
fn test_integers() {
    assert_eq!(tok_literals("42 0 100"), vec!["42", "0", "100"]);
    assert_eq!(tok_types("42"), vec![TokenType::Integer]);
}

#[test]
fn test_hex_octal() {
    assert_eq!(tok_types("0xFF"), vec![TokenType::Integer]);
    assert_eq!(tok_literals("0xFF"), vec!["0xFF"]);
    assert_eq!(tok_types("0o77"), vec![TokenType::Integer]);
    assert_eq!(tok_literals("0o77"), vec!["0o77"]);
}

#[test]
fn test_floats() {
    assert_eq!(tok_types("3.14"), vec![TokenType::Float]);
    assert_eq!(tok_types("1e10"), vec![TokenType::Float]);
    assert_eq!(tok_types("2.5E-3"), vec![TokenType::Float]);
    assert_eq!(tok_types(".5"), vec![TokenType::Float]);
    // An explicit double may carry a trailing `d`/`D` suffix.
    assert_eq!(tok_types(".4d"), vec![TokenType::Float]);
    assert_eq!(tok_types("1e4D"), vec![TokenType::Float]);
}

#[test]
fn test_strings() {
    let tokens = tokenize("'hello' \"world\"");
    assert_eq!(tokens.len(), 2);
    assert_eq!(tokens[0].ty, TokenType::String);
    assert_eq!(tokens[0].literal, "hello");
    assert_eq!(tokens[1].ty, TokenType::String);
    assert_eq!(tokens[1].literal, "world");
}

#[test]
fn test_string_escapes() {
    let tokens = tokenize(r"'hello\nworld'");
    assert_eq!(tokens[0].literal, "hello\nworld");

    let tokens = tokenize(r"'tab\there'");
    assert_eq!(tokens[0].literal, "tab\there");

    let tokens = tokenize(r"'escaped\\slash'");
    assert_eq!(tokens[0].literal, "escaped\\slash");
}

#[test]
fn test_unicode_escape() {
    let tokens = tokenize(r"'\u0041'");
    assert_eq!(tokens[0].literal, "A");
}

#[test]
fn test_interpolation_escapes_match_string_escapes() {
    // Round-3 CIP-261 design: the InterpString token carries escapes RAW
    // (both text and embed regions) — `parse_interp_body` resolves text
    // escapes with an unambiguous two-byte lookahead. The lexer-level
    // assertion is therefore the raw body; resolution is covered by the
    // parser tests (`test_interp_embed_escaped_quote_kept_raw` etc.).
    let tokens = tokenize(r"s'a\nb\tc'");
    assert_eq!(tokens[0].ty, TokenType::InterpString);
    assert_eq!(tokens[0].literal, r"a\nb\tc");

    let tokens = tokenize(r"s'\u0041{1}'");
    assert_eq!(tokens[0].literal, r"\u0041{1}");

    let tokens = tokenize(r"s'\{x\}'");
    assert_eq!(tokens[0].literal, r"\{x\}");

    // An unknown escape is preserved verbatim, like a plain string.
    let tokens = tokenize(r"s'\q'");
    assert_eq!(tokens[0].literal, "\\q");
}

#[test]
fn test_backtick_identifier() {
    let tokens = tokenize("`my label`");
    assert_eq!(tokens.len(), 1);
    assert_eq!(tokens[0].ty, TokenType::Identifier);
    assert_eq!(tokens[0].literal, "my label");
}

#[test]
fn test_backtick_identifier_doubled_backtick_is_escaped() {
    // A doubled backtick inside a backtick-quoted identifier collapses to a
    // single literal backtick, mirroring the formatter's `cypher_identifier`
    // escaping so `.dump` / `.mode cypher` output round-trips (the bug: the
    // lexer used to terminate at the first backtick, so `` `a``b` `` lexed as
    // `a` followed by a stray `b`).
    let tokens = tokenize("`a``b`");
    assert_eq!(tokens.len(), 1, "one identifier token: {tokens:?}");
    assert_eq!(tokens[0].ty, TokenType::Identifier);
    assert_eq!(tokens[0].literal, "a`b");

    // A trailing escaped pair, and a leading one.
    assert_eq!(tokenize("`x```")[0].literal, "x`");
    assert_eq!(tokenize("```y`")[0].literal, "`y");

    // A lone backtick still terminates: `` `a`b `` is `a` then bare `b`.
    let two = tokenize("`a`b");
    assert_eq!(two.len(), 2);
    assert_eq!(two[0].literal, "a");
    assert_eq!(two[1].literal, "b");
}

#[test]
fn test_parameter() {
    let tokens = tokenize("$name $value");
    assert_eq!(tokens.len(), 2);
    assert_eq!(tokens[0].ty, TokenType::Parameter);
    assert_eq!(tokens[0].literal, "name");
    assert_eq!(tokens[1].literal, "value");
}

#[test]
fn test_line_comment() {
    let _tokens = tokenize("MATCH // this is a comment\nRETURN");
    assert_eq!(
        tok_types("MATCH // comment\nRETURN"),
        vec![TokenType::MATCH, TokenType::RETURN]
    );
}

#[test]
fn test_block_comment() {
    assert_eq!(
        tok_types("MATCH /* block */ RETURN"),
        vec![TokenType::MATCH, TokenType::RETURN]
    );
}

#[test]
fn test_position_tracking() {
    let tokens = tokenize("MATCH (n)\nRETURN n");
    assert_eq!(tokens[0].line, 1);
    assert_eq!(tokens[0].col, 1);
    // RETURN should be on line 2
    let ret_tok = tokens.iter().find(|t| t.ty == TokenType::RETURN).unwrap();
    assert_eq!(ret_tok.line, 2);
    assert_eq!(ret_tok.col, 1);
}

#[test]
fn test_full_query() {
    let tokens = tokenize("MATCH (n:Person {name: 'Alice'})-[r:KNOWS]->(m) RETURN m.name");
    assert!(tokens.len() > 10);
    assert_eq!(tokens[0].ty, TokenType::MATCH);
}

#[test]
fn test_unterminated_string() {
    let tokens = tokenize("'hello");
    assert_eq!(tokens[0].ty, TokenType::Error);
    assert_eq!(tokens[0].literal, "unterminated string");
}

#[test]
fn test_interpolation_embed_escapes_pass_through_raw() {
    // Inside a `{...}` embed the body is re-lexed as an expression, so
    // escapes must reach it verbatim: `\'` stays `\'`, not `'`.
    // (Round-3 review: silently dropped by the test-module move; restored.)
    let tokens = tokenize(r"s'{coalesce('a\'b','c')}'");
    assert_eq!(tokens[0].ty, TokenType::InterpString);
    assert_eq!(tokens[0].literal, r"{coalesce('a\'b','c')}");

    // `\{`/`\}` inside an embed also pass through raw.
    let tokens = tokenize(r"s'{coalesce('x\{y','z')}'");
    assert_eq!(tokens[0].literal, r"{coalesce('x\{y','z')}");
}

#[test]
fn test_interpolation_escapes_pass_through_raw() {
    // `read_interpolation` leaves *every* escape unresolved in the token
    // body -- the interpolation parser decodes text segments later. That
    // is what keeps `\\` + `{` distinguishable from `\{` (a lexer-resolved
    // `\\` would look like the start of `\{`).
    let tokens = tokenize(r"s'a\nb\tc'");
    assert_eq!(tokens[0].ty, TokenType::InterpString);
    assert_eq!(tokens[0].literal, r"a\nb\tc");

    let tokens = tokenize(r"s'\u0041{1}'");
    assert_eq!(tokens[0].literal, r"\u0041{1}");

    let tokens = tokenize(r"s'\{x\}'");
    assert_eq!(tokens[0].literal, r"\{x\}");

    // An unknown escape is also preserved verbatim.
    let tokens = tokenize(r"s'\q'");
    assert_eq!(tokens[0].literal, r"\q");
}

#[test]
fn test_interpolation_escaped_backslash_before_embed() {
    // Round-3 review: `\\` + `{` is an escaped backslash followed by an
    // embed, distinct from `\{` (an escaped literal brace). The lexer must
    // keep them apart in the raw body.
    assert_eq!(tokenize(r"s'\\{1}'")[0].literal, r"\\{1}");
    assert_eq!(tokenize(r"s'\{1}'")[0].literal, r"\{1}");
    assert_eq!(tokenize(r"s'a\\b{x}'")[0].literal, r"a\\b{x}");
}

#[test]
fn test_interpolation_backtick_name_in_embed() {
    // A backtick-quoted identifier inside an embed is inert: braces and
    // quotes within it must not close the embed or open a string.
    assert_eq!(tokenize(r"s'{`a}b`}'")[0].ty, TokenType::InterpString);
    assert_eq!(tokenize(r"s'{`a}b`}'")[0].literal, r"{`a}b`}");
    assert_eq!(tokenize(r#"s'{`x'y`}'"#)[0].literal, r"{`x'y`}");
}

#[test]
fn test_backtick_identifier_non_ascii_roundtrip() {
    // Non-ASCII bytes must be decoded as UTF-8, not widened per-byte
    // (which would mojibake `Größe` into `Größe` and break the
    // dump/re-parse round-trip).
    let tokens = tokenize("`Größe`");
    assert_eq!(tokens.len(), 1);
    assert_eq!(tokens[0].ty, TokenType::Identifier);
    assert_eq!(tokens[0].literal, "Größe");
    assert_eq!(tokens[0].literal.as_bytes(), "Größe".as_bytes());
}

#[test]
fn test_unicode_escape_invalid_codepoint_is_error() {
    // Surrogate codepoints are not valid `char`s; they must produce a lex
    // error rather than being silently dropped from the string.
    for input in [r"'\uD800'", r"'\uDFFF'"] {
        let tokens = tokenize(input);
        assert_eq!(tokens[0].ty, TokenType::Error, "{input}");
        assert!(
            tokens[0].literal.contains("invalid unicode escape"),
            "{input}: {}",
            tokens[0].literal
        );
    }
}

#[test]
fn test_unicode_escape_window_cut_mid_character_is_error() {
    // The `\u` operand window is a raw four-byte cut and can end
    // mid-character (`'\u00€'` yields `30 30 E2 82`): it must be one
    // `InvalidUnicodeLiteral` error naming the bytes found — never a
    // fabricated empty operand silently substituted for them.
    let tokens = tokenize("'\u{5c}u00€'");
    assert_eq!(tokens[0].ty, TokenType::Error);
    assert!(
        tokens[0].literal.contains("invalid unicode escape"),
        "{}",
        tokens[0].literal
    );
    assert_eq!(tokens[0].detail, Some("InvalidUnicodeLiteral"));
    // A valid-UTF-8 window of non-hex digits reports the window itself,
    // exactly as the radix rejection always did.
    let tokens = tokenize(r"'\uGGGG'");
    assert_eq!(tokens[0].ty, TokenType::Error);
    assert_eq!(
        tokens[0].literal,
        r"invalid unicode escape sequence: \uGGGG"
    );
    assert_eq!(tokens[0].detail, Some("InvalidUnicodeLiteral"));
}

#[test]
fn test_unicode_escape_surrogate_pair_is_one_codepoint() {
    // A UTF-16 surrogate pair escape names one supplementary code point
    // (U+10000, the TCK's `'𐀀'`); the lone-surrogate rejection above still
    // covers a high surrogate with no low pair.
    let tokens = tokenize("'\\uD800\\uDC00'");
    assert_eq!(tokens.len(), 1);
    assert_eq!(tokens[0].ty, TokenType::String);
    assert_eq!(tokens[0].literal, "\u{10000}");
    // A high surrogate followed by anything but a low surrogate stays an
    // error (the second escape alone is not a pair).
    for input in [r"'\uD800\uD800'", r"'\uD800x'", r"'\uDC00\uD800'"] {
        let tokens = tokenize(input);
        assert_eq!(tokens[0].ty, TokenType::Error, "{input}");
    }
}

#[test]
fn test_escape_fallback_preserves_non_ascii() {
    // An unknown escape keeps the backslash and the escaped character;
    // a non-ASCII escaped character must survive byte-identically.
    let tokens = tokenize("'\\é'");
    assert_eq!(tokens.len(), 1);
    assert_eq!(tokens[0].ty, TokenType::String);
    assert_eq!(tokens[0].literal, "\\é");

    // Mixed with a recognized escape.
    let tokens = tokenize("'é\\n'");
    assert_eq!(tokens[0].literal, "é\n");
}

#[test]
fn test_unterminated_block_comment_is_error() {
    // An unterminated `/*` must not silently swallow the rest of the
    // input — `RETURN 1 /* DETACH DELETE n` must be rejected, not lexed
    // as plain `RETURN 1`.
    let tokens = tokenize("RETURN 1 /* DETACH DELETE n");
    assert_eq!(tokens.len(), 3);
    assert_eq!(tokens[0].ty, TokenType::RETURN);
    assert_eq!(tokens[1].ty, TokenType::Integer);
    assert_eq!(tokens[2].ty, TokenType::Error);
    assert_eq!(tokens[2].literal, "unterminated block comment");

    // A comment terminated at EOF is still fine.
    assert_eq!(
        tok_types("RETURN 1 /* done */"),
        vec![TokenType::RETURN, TokenType::Integer]
    );
}

#[test]
fn test_unicode_identifier_lexes_as_one_token() {
    // ADR-0182: a non-ASCII identifier start opens one identifier token and
    // identifier parts (letters, `Pc`, `Mn`/`Mc`, `Nd`, and `Sc` outside
    // `Pattern_Syntax`) continue it -- the whole run is a single token,
    // never split per UTF-8 byte.
    for (query, ident) in [
        ("RETURN 1 AS Éx", "Éx"),
        ("RETURN 1 AS xÉ", "xÉ"),
        ("RETURN 1 AS xé", "xé"),
        ("RETURN 1 AS x€", "x€"),               // Sc outside Pattern_Syntax
        ("RETURN 1 AS x名前", "x名前"),         // CJK letters
        ("RETURN 1 AS x\u{0301}", "x\u{0301}"), // combining mark part
        ("RETURN 1 AS x\u{08BE}", "x\u{08BE}"), // Unicode 13 addition
        ("RETURN 1 AS \u{203F}x", "\u{203F}x"), // Pc start (not XID_Start)
        ("RETURN 1 AS \u{017F}KIP", "\u{017F}KIP"), // never a keyword
    ] {
        let tokens = tokenize(query);
        let last = tokens.last().expect("at least one token");
        assert_eq!(last.ty, TokenType::Identifier, "{query:?}: {tokens:?}");
        assert_eq!(last.literal, ident, "{query:?}");
    }

    // Parameter names follow the same rule.
    let tokens = tokenize("RETURN $名前 AS v");
    let param = tokens
        .iter()
        .find(|t| t.ty == TokenType::Parameter)
        .unwrap_or_else(|| panic!("expected a parameter token: {tokens:?}"));
    assert_eq!(param.literal, "名前");
}

#[test]
fn test_unicode_identifier_start_rejects_non_start_characters() {
    // ADR-0182: 😀 (So, supplementary plane), § (Po ∩ Pattern_Syntax), €
    // (Sc: a part, never a start) and U+00B7 (Other_ID_Continue) are not
    // identifier starts; each is ONE InvalidUnicodeCharacter error token.
    for (query, ch) in [
        ("RETURN 1 AS 😀x", '😀'),
        ("RETURN 1 AS §x", '§'),
        ("RETURN 1 AS €x", '€'),
        ("RETURN 1 AS ·x", '·'),
        ("RETURN 1 AS x·", '·'), // not a part either
    ] {
        let tokens = tokenize(query);
        let err = tokens
            .iter()
            .find(|t| t.ty == TokenType::Error)
            .unwrap_or_else(|| panic!("{query:?}: expected an error token"));
        assert_eq!(err.detail, Some("InvalidUnicodeCharacter"), "{query:?}");
        assert!(err.literal.contains(ch), "{query:?}: {:?}", err.literal);
        // The whole multi-byte sequence is consumed: the `x` on the other
        // side of the rejected character still lexes as an identifier.
        assert!(
            tokens
                .iter()
                .any(|t| t.ty == TokenType::Identifier && t.literal.contains('x')),
            "{query:?}: {tokens:?}"
        );
    }
}

#[test]
fn test_parameter_start_rejects_part_only_characters() {
    // A parameter's first character is an identifier start or an ASCII digit
    // (ADR-0182's Consequences prose, confirmed against the live Neo4j
    // reference — LABS-209): `€`-class part-only starts are rejected with the
    // reference's error shape, and the offending run is consumed as one error
    // token, not a cascade.
    for (query, bad) in [("RETURN $€ AS v", "€"), ("RETURN $€a AS v", "€a")] {
        let tokens = tokenize(query);
        let err = tokens
            .iter()
            .find(|t| t.ty == TokenType::Error)
            .unwrap_or_else(|| panic!("{query:?} must produce an error token"));
        assert_eq!(err.detail, Some("InvalidParameterName"), "{query:?}");
        assert!(
            err.literal.contains(bad),
            "{query:?}: literal must name {bad:?}: {:?}",
            err.literal
        );
        assert!(
            err.literal
                .contains("expected an identifier or an integer value"),
            "{query:?}: message shape: {:?}",
            err.literal
        );
    }
}

#[test]
fn test_parameter_start_accepts_digits_and_identifier_starts() {
    // Neo4j 5.26 accepts `$1`, `$1x` and Unicode identifier starts
    // (`$名前`, `$café`) bare (LABS-209's probe); backticks are only needed
    // for names that are not identifier-shaped at all.
    for query in [
        "RETURN $1 AS v",
        "RETURN $1x AS v",
        "RETURN $名前 AS v",
        "RETURN $café AS v",
    ] {
        let tokens = tokenize(query);
        assert!(
            !tokens.iter().any(|t| t.ty == TokenType::Error),
            "{query:?} must lex clean: {tokens:?}"
        );
    }
}

#[test]
fn test_parameter_dynamic_opener_lexes_clean() {
    // The `(` after `$` is the dynamic-expression opener (`$(expr)`, e.g.
    // `n IS LABELED $('A')`) — a separate contract from the
    // identifier-start/digit acceptance above: the lexer emits the
    // intentionally empty parameter token whose expansion the parser owns.
    let tokens = tokenize("MATCH (n:$(x)) RETURN n");
    assert!(
        !tokens.iter().any(|t| t.ty == TokenType::Error),
        "the dynamic opener must lex clean: {tokens:?}"
    );
}

#[test]
fn test_bare_parameter_rejected() {
    // A `$` with no name (EOF or whitespace next) is an error token naming
    // what was expected, never the old silent empty-named parameter.
    let tokens = tokenize("RETURN $");
    let err = tokens
        .iter()
        .find(|t| t.ty == TokenType::Error)
        .unwrap_or_else(|| panic!("bare $ must produce an error token"));
    assert_eq!(err.detail, Some("InvalidParameterName"));
    assert!(
        err.literal
            .contains("expected an identifier or an integer value"),
        "message shape: {:?}",
        err.literal
    );
}

// ---- ASCII fast path (SWAR batches, ADR-0505) ----

#[test]
fn test_ascii_fast_path_long_identifier_runs() {
    // Runs of every batch shape stay one token with exact text: the
    // single-byte tail, the exactly-one-batch run, and runs that cross
    // the eight-byte boundary (two full batches plus a tail).
    for name in [
        "n",
        "abcdefgh",
        "abcdefghi",
        "abcdefghijklmnopqrst",
        "a1_b2_c3_d4_e5_f6_g7",
    ] {
        let query = format!("RETURN 1 AS {name}");
        let tokens = tokenize(&query);
        let last = tokens
            .last()
            .unwrap_or_else(|| panic!("{query:?}: at least one token"));
        assert_eq!(last.ty, TokenType::Identifier, "{query:?}: {tokens:?}");
        assert_eq!(last.literal, name, "{query:?}");
    }
}

#[test]
fn test_ascii_fast_path_stops_at_non_ascii_and_resumes() {
    // The batch consumes the ASCII stretch, the first byte >= 0x80 hands
    // the character to the decoded path, and batching resumes after it —
    // one token, exact text, positions unchanged (ADR-0182 behavior on
    // the slow path, byte-for-byte).
    for name in [
        "aaaaaaaaéaaaaaaaa",
        "éaaaaaaaaaaaaaaaa",
        "aaaaaaaaaaaaaaaaé",
        "aé_bÉ1€x",
    ] {
        let query = format!("RETURN 1 AS {name}");
        let tokens = tokenize(&query);
        let last = tokens
            .last()
            .unwrap_or_else(|| panic!("{query:?}: at least one token"));
        assert_eq!(last.ty, TokenType::Identifier, "{query:?}: {tokens:?}");
        assert_eq!(last.literal, name, "{query:?}");
    }

    // Parameter names take the same run scanner.
    let tokens = tokenize("RETURN $aaaaaaaaébbbb AS v");
    let param = tokens
        .iter()
        .find(|t| t.ty == TokenType::Parameter)
        .unwrap_or_else(|| panic!("expected a parameter token: {tokens:?}"));
    assert_eq!(param.literal, "aaaaaaaaébbbb");
}

#[test]
fn test_ascii_fast_path_positions_after_batched_runs() {
    // A long whitespace run (two batches) with newlines, tabs, spaces and
    // a CR: the batched skip must leave line/column exactly where
    // byte-at-a-time advance did.
    let tokens = tokenize("MATCH (n)\n  \t \r\n   RETURN n.name");
    let ret = tokens
        .iter()
        .find(|t| t.ty == TokenType::RETURN)
        .unwrap_or_else(|| panic!("expected RETURN: {tokens:?}"));
    assert_eq!((ret.line, ret.col), (3, 4), "{tokens:?}");

    let tokens = tokenize("MATCH (n)\n          RETURN");
    let ret = tokens
        .iter()
        .find(|t| t.ty == TokenType::RETURN)
        .unwrap_or_else(|| panic!("expected RETURN: {tokens:?}"));
    assert_eq!((ret.line, ret.col), (2, 11), "{tokens:?}");

    // The token after a long identifier batch: the column counts bytes,
    // one per identifier-part byte, exactly as per-byte advance did.
    let tokens = tokenize("RETURN abcdefghijklmnop, 1");
    let int = tokens
        .iter()
        .find(|t| t.ty == TokenType::Integer)
        .unwrap_or_else(|| panic!("expected the integer: {tokens:?}"));
    assert_eq!((int.line, int.col), (1, 26), "{tokens:?}");

    // A run that went through the decoded path advances the column by the
    // character's byte length, same as before the fast path.
    let tokens = tokenize("RETURN aé, 1");
    let int = tokens
        .iter()
        .find(|t| t.ty == TokenType::Integer)
        .unwrap_or_else(|| panic!("expected the integer: {tokens:?}"));
    assert_eq!((int.line, int.col), (1, 13), "{tokens:?}");
}

#[test]
fn test_ascii_fast_path_heavy_whitespace_matches_single_spaces() {
    // Whitespace amount and kind never change the token stream: types and
    // literals of the wide-run form equal the single-space form (only
    // positions differ).
    let thin = tokenize("MATCH (n:Person)-[:KNOWS]->(m) WHERE n.age > 21 RETURN m.name");
    let wide = tokenize(
        "MATCH   (n:Person)\n\t-\t[:KNOWS]->\r\n  (m)   WHERE\tn.age\t>  21  RETURN m.name",
    );
    let shape = |ts: &[Token<'_>]| {
        ts.iter()
            .map(|t| (t.ty, t.literal.to_string()))
            .collect::<Vec<_>>()
    };
    assert_eq!(shape(&thin), shape(&wide));
    assert!(
        !wide.iter().any(|t| t.ty == TokenType::Error),
        "the wide form must lex clean: {wide:?}"
    );
}