1use super::unicode_ops::unicode_op_kind;
10use crate::keywords::keyword_table;
11use crate::tokens::token_table;
12
13mod operators;
14
15macro_rules! define_tok_kind {
18 ($($(#[$meta:meta])* $tok:ident $syn:ident,)*) => {
19#[derive(Debug, Clone, Copy, PartialEq, Eq)]
22pub(crate) enum TokKind {
23 $($(#[$meta])* $tok,)*
24
25 UniArrow,
33 UniComparison,
34 UniColon,
35 UniPlus,
36 UniTimes,
37 UniPower,
38}
39 };
40}
41
42token_table!(define_tok_kind);
43
44impl TokKind {
45 pub(crate) fn is_trivia(self) -> bool {
48 matches!(
49 self,
50 TokKind::Whitespace | TokKind::Newline | TokKind::Comment | TokKind::BlockComment
51 )
52 }
53}
54
55#[derive(Debug, Clone, PartialEq, Eq)]
56pub(crate) struct Token<'src> {
57 pub(crate) kind: TokKind,
58 pub(crate) text: &'src str,
59 pub(crate) start: usize,
60 pub(crate) end: usize,
61}
62
63pub(crate) fn lex(input: &str) -> Vec<Token<'_>> {
67 Lexer::new(input).run()
68}
69
70#[derive(Debug, Clone, Copy, PartialEq, Eq)]
75enum Mode {
76 Str {
78 triple: bool,
79 raw: bool,
82 prefixed: bool,
83 },
84 Cmd {
86 triple: bool,
87 raw: bool,
88 prefixed: bool,
89 },
90 Interp { depth: usize },
92}
93
94pub(super) struct Lexer<'a> {
95 input: &'a str,
96 pub(super) bytes: &'a [u8],
97 pub(super) pos: usize,
98 tokens: Vec<Token<'a>>,
99 mode_stack: Vec<Mode>,
100}
101
102impl<'a> Lexer<'a> {
103 fn new(input: &'a str) -> Self {
104 Self {
105 input,
106 bytes: input.as_bytes(),
107 pos: 0,
108 tokens: Vec::new(),
109 mode_stack: Vec::new(),
110 }
111 }
112
113 fn run(mut self) -> Vec<Token<'a>> {
114 while self.pos < self.bytes.len() {
115 self.next_token();
116 }
117 self.tokens
118 }
119
120 fn peek(&self, ahead: usize) -> Option<u8> {
121 self.bytes.get(self.pos + ahead).copied()
122 }
123
124 fn push(&mut self, kind: TokKind, start: usize, end: usize) {
125 self.tokens.push(Token {
126 kind,
127 text: &self.input[start..end],
128 start,
129 end,
130 });
131 }
132
133 fn push_op(&mut self, kind: TokKind, start: usize) {
139 if op_takes_suffix(kind) {
140 while self.pos < self.bytes.len() {
141 let c = self.char_at(self.pos);
142 if is_op_suffix_char(c) {
143 self.pos += c.len_utf8();
144 } else {
145 break;
146 }
147 }
148 }
149 self.push(kind, start, self.pos);
150 }
151
152 fn next_token(&mut self) {
153 if matches!(
156 self.mode_stack.last(),
157 Some(Mode::Str { .. } | Mode::Cmd { .. })
158 ) {
159 self.lex_in_string_mode();
160 return;
161 }
162
163 let start = self.pos;
164 let b = self.bytes[self.pos];
165
166 if matches!(self.mode_stack.last(), Some(Mode::Interp { .. })) {
169 if b == b'(' {
170 self.pos += 1;
171 self.push(TokKind::LParen, start, self.pos);
172 if let Some(Mode::Interp { depth }) = self.mode_stack.last_mut() {
173 *depth += 1;
174 }
175 return;
176 }
177 if b == b')' {
178 self.pos += 1;
179 self.push(TokKind::RParen, start, self.pos);
180 if matches!(self.mode_stack.last(), Some(Mode::Interp { depth }) if *depth == 1) {
181 self.mode_stack.pop();
182 } else if let Some(Mode::Interp { depth }) = self.mode_stack.last_mut() {
183 *depth -= 1;
184 }
185 return;
186 }
187 }
188
189 match b {
190 b' ' | b'\t' => self.lex_whitespace(start),
191 b'\r' | b'\n' => self.lex_newline(start),
192 b'#' => self.lex_comment(start),
193 b'"' => self.lex_open_string(start, false),
194 b'`' => self.lex_open_cmd(start, false),
195 b'\'' if self.prev_ends_value() || self.prev_is_dot() => {
196 self.pos += 1;
197 self.push_op(TokKind::Transpose, start);
198 }
199 b'\'' => self.lex_char_literal(start),
200 b'0'..=b'9' => self.lex_number(start),
201 b'.' if self.peek(1).is_some_and(|c| c.is_ascii_digit()) => self.lex_number(start),
202 _ => {
203 if is_ident_start(self.char_at(self.pos)) {
204 self.lex_ident_or_keyword(start);
205 } else {
206 self.lex_operator_or_unknown(start);
207 }
208 }
209 }
210 }
211
212 pub(super) fn char_at(&self, at: usize) -> char {
214 self.input[at..].chars().next().unwrap_or('\0')
215 }
216
217 fn lex_whitespace(&mut self, start: usize) {
218 while matches!(self.peek(0), Some(b' ' | b'\t')) {
219 self.pos += 1;
220 }
221 self.push(TokKind::Whitespace, start, self.pos);
222 }
223
224 fn lex_newline(&mut self, start: usize) {
225 match self.peek(0) {
227 Some(b'\r') if self.peek(1) == Some(b'\n') => self.pos += 2,
228 _ => self.pos += 1,
229 }
230 self.push(TokKind::Newline, start, self.pos);
231 }
232
233 fn lex_comment(&mut self, start: usize) {
234 if self.peek(1) == Some(b'=') {
235 self.lex_block_comment(start);
236 return;
237 }
238 self.pos += 1;
240 while !matches!(self.peek(0), Some(b'\n' | b'\r') | None) {
241 self.pos += 1;
242 }
243 self.push(TokKind::Comment, start, self.pos);
244 }
245
246 fn lex_block_comment(&mut self, start: usize) {
249 self.pos += 2; let mut depth = 1usize;
251 while depth > 0 && self.pos < self.bytes.len() {
252 match (self.peek(0), self.peek(1)) {
253 (Some(b'#'), Some(b'=')) => {
254 self.pos += 2;
255 depth += 1;
256 }
257 (Some(b'='), Some(b'#')) => {
258 self.pos += 2;
259 depth -= 1;
260 }
261 _ => self.pos += 1,
262 }
263 }
264 self.push(TokKind::BlockComment, start, self.pos);
265 }
266
267 fn lex_open_string(&mut self, start: usize, prefixed: bool) {
272 let triple = self.peek(1) == Some(b'"') && self.peek(2) == Some(b'"');
273 self.pos += if triple { 3 } else { 1 };
274 self.push(TokKind::StringDelimOpen, start, self.pos);
275 self.mode_stack.push(Mode::Str {
276 triple,
277 raw: prefixed,
278 prefixed,
279 });
280 }
281
282 fn lex_open_cmd(&mut self, start: usize, prefixed: bool) {
284 let triple = self.peek(1) == Some(b'`') && self.peek(2) == Some(b'`');
285 self.pos += if triple { 3 } else { 1 };
286 self.push(TokKind::CmdDelimOpen, start, self.pos);
287 self.mode_stack.push(Mode::Cmd {
288 triple,
289 raw: prefixed,
290 prefixed,
291 });
292 }
293
294 fn lex_in_string_mode(&mut self) {
297 let frame = *self.mode_stack.last().expect("string mode frame");
298 let (quote, triple, raw, prefixed) = match frame {
299 Mode::Str {
300 triple,
301 raw,
302 prefixed,
303 } => (b'"', triple, raw, prefixed),
304 Mode::Cmd {
305 triple,
306 raw,
307 prefixed,
308 } => (b'`', triple, raw, prefixed),
309 Mode::Interp { .. } => unreachable!("lex_in_string_mode called in interp mode"),
310 };
311 let close_kind = if quote == b'"' {
312 TokKind::StringDelimClose
313 } else {
314 TokKind::CmdDelimClose
315 };
316
317 let start = self.pos;
318
319 if self.at_close_delim(quote, triple) {
321 self.pos += if triple { 3 } else { 1 };
322 self.push(close_kind, start, self.pos);
323 self.mode_stack.pop();
324 if prefixed {
325 self.lex_suffix();
326 }
327 return;
328 }
329
330 while self.pos < self.bytes.len() {
335 if raw && self.peek(0) == Some(b'\\') {
340 let mut run = 0;
341 while self.peek(run) == Some(b'\\') {
342 run += 1;
343 }
344 if self.peek(run) == Some(quote) && run % 2 == 1 {
345 self.pos += run + 1;
346 } else {
347 self.pos += run;
348 }
349 continue;
350 }
351 if self.at_close_delim(quote, triple) {
352 break;
353 }
354 if !raw && self.peek(0) == Some(b'$') && self.is_interp_start(1) {
355 break;
356 }
357 self.consume_body_byte(raw);
358 }
359
360 if self.pos > start {
361 self.push(TokKind::StringContent, start, self.pos);
362 }
363
364 if self.at_close_delim(quote, triple) {
366 let delim_start = self.pos;
367 self.pos += if triple { 3 } else { 1 };
368 self.push(close_kind, delim_start, self.pos);
369 self.mode_stack.pop();
370 if prefixed {
371 self.lex_suffix();
372 }
373 } else if !raw && self.peek(0) == Some(b'$') && self.is_interp_start(1) {
374 self.lex_interp_sigil();
375 } else {
376 self.mode_stack.pop();
379 }
380 }
381
382 fn at_close_delim(&self, quote: u8, triple: bool) -> bool {
384 if triple {
385 self.peek(0) == Some(quote)
386 && self.peek(1) == Some(quote)
387 && self.peek(2) == Some(quote)
388 } else {
389 self.peek(0) == Some(quote)
390 }
391 }
392
393 fn is_interp_start(&self, ahead: usize) -> bool {
396 match self.peek(ahead) {
397 Some(b'(') => true,
398 Some(_) => is_ident_start(self.char_at(self.pos + ahead)),
399 None => false,
400 }
401 }
402
403 fn lex_interp_sigil(&mut self) {
406 let dollar = self.pos;
407 self.pos += 1;
408 self.push(TokKind::Dollar, dollar, self.pos);
409 if self.peek(0) == Some(b'(') {
410 let paren = self.pos;
411 self.pos += 1;
412 self.push(TokKind::LParen, paren, self.pos);
413 self.mode_stack.push(Mode::Interp { depth: 1 });
414 } else {
415 self.lex_interp_ident();
418 }
419 }
420
421 fn consume_body_byte(&mut self, raw: bool) {
428 if !raw && self.peek(0) == Some(b'\\') && self.pos + 1 < self.bytes.len() {
429 if self.peek(1) == Some(b'\r') && self.peek(2) == Some(b'\n') {
430 self.pos += 3;
431 } else {
432 self.pos += 1 + self.char_at(self.pos + 1).len_utf8();
433 }
434 } else {
435 self.pos += self.char_at(self.pos).len_utf8();
436 }
437 }
438
439 fn lex_suffix(&mut self) {
444 let start = self.pos;
445 if matches!(self.peek(0), Some(c) if c.is_ascii_alphabetic()) {
446 self.pos += 1;
447 while matches!(self.peek(0), Some(c) if c.is_ascii_alphanumeric()) {
448 self.pos += 1;
449 }
450 }
451 if self.pos > start {
452 self.push(TokKind::StringSuffix, start, self.pos);
453 }
454 }
455
456 fn prev_ends_value(&self) -> bool {
462 matches!(
463 self.tokens.last().map(|t| t.kind),
464 Some(
465 TokKind::Ident
466 | TokKind::Integer
467 | TokKind::Float
468 | TokKind::Char
469 | TokKind::TrueKw
470 | TokKind::FalseKw
471 | TokKind::RParen
472 | TokKind::RBracket
473 | TokKind::RBrace
474 | TokKind::StringDelimClose
475 | TokKind::CmdDelimClose
476 | TokKind::StringSuffix
477 | TokKind::Transpose
478 )
479 )
480 }
481
482 fn prev_is_dot(&self) -> bool {
487 self.tokens
488 .iter()
489 .rev()
490 .find(|t| !matches!(t.kind, TokKind::Whitespace | TokKind::BlockComment))
491 .map(|t| t.kind)
492 == Some(TokKind::Dot)
493 }
494
495 fn lex_char_literal(&mut self, start: usize) {
504 let mut idx = self.pos + 1;
512 let mut found = false;
513 while idx < self.bytes.len() {
514 match self.bytes[idx] {
515 b'\'' => {
516 found = true;
517 break;
518 }
519 b'\\' => {
520 idx += 1;
521 if idx < self.bytes.len() {
522 idx += self.char_at(idx).len_utf8();
523 }
524 }
525 _ => idx += self.char_at(idx).len_utf8(),
526 }
527 }
528
529 self.pos = if found { idx + 1 } else { idx };
533 self.push(TokKind::Char, start, self.pos);
534 }
535
536 fn lex_number(&mut self, start: usize) {
537 if self.peek(0) == Some(b'0') {
541 match self.peek(1) {
542 Some(b'x') => {
543 self.pos += 2;
544 let int_digits = self.consume_digits(|c| c.is_ascii_hexdigit());
554 let mut had_dot = false;
557 let mut frac_digits = false;
558 if self.peek(0) == Some(b'.') && self.peek(1) != Some(b'.') {
559 had_dot = true;
560 self.pos += 1;
561 frac_digits = self.consume_digits(|c| c.is_ascii_hexdigit());
562 }
563 let mut had_exponent = false;
564 let mut exp_digits = false;
565 if matches!(self.peek(0), Some(b'p' | b'P')) {
566 had_exponent = true;
567 self.pos += 1;
568 if matches!(self.peek(0), Some(b'+' | b'-')) {
569 self.pos += 1;
570 }
571 exp_digits = self.consume_digits(|c| c.is_ascii_digit());
572 }
573 let has_mantissa = int_digits || frac_digits;
574 let kind = if had_exponent {
575 if has_mantissa && exp_digits {
576 TokKind::Float
577 } else {
578 TokKind::ErrorInvalidNumber
579 }
580 } else if had_dot {
581 TokKind::ErrorHexFloatNoP
582 } else if int_digits {
583 TokKind::HexInt
584 } else {
585 TokKind::ErrorInvalidNumber
586 };
587 self.push(kind, start, self.pos);
588 return;
589 }
590 Some(b'o') => {
591 self.pos += 2;
592 self.consume_digits(|c| (b'0'..=b'7').contains(&c));
593 self.push(TokKind::OctInt, start, self.pos);
594 return;
595 }
596 Some(b'b') => {
597 self.pos += 2;
598 self.consume_digits(|c| matches!(c, b'0' | b'1'));
599 self.push(TokKind::BinInt, start, self.pos);
600 return;
601 }
602 _ => {}
603 }
604 }
605
606 let mut is_float = false;
607 let mut is_f32 = false;
608 self.consume_digits(|c| c.is_ascii_digit());
609 if self.peek(0) == Some(b'.') && self.peek(1) != Some(b'.') {
613 is_float = true;
614 self.pos += 1;
615 self.consume_digits(|c| c.is_ascii_digit());
616 }
617 if matches!(self.peek(0), Some(b'e' | b'E' | b'f'))
623 && matches!(self.peek(1), Some(c) if c.is_ascii_digit() || c == b'+' || c == b'-')
624 {
625 if self.peek(0) == Some(b'f') {
626 is_f32 = true;
627 } else {
628 is_float = true;
629 }
630 self.pos += 1;
631 if matches!(self.peek(0), Some(b'+' | b'-')) {
632 self.pos += 1;
633 }
634 while matches!(self.peek(0), Some(c) if c.is_ascii_digit()) {
635 self.pos += 1;
636 }
637 }
638
639 let kind = if is_f32 {
640 TokKind::Float32
641 } else if is_float {
642 TokKind::Float
643 } else {
644 TokKind::Integer
645 };
646 self.push(kind, start, self.pos);
647 }
648
649 fn consume_digits(&mut self, is_digit: impl Fn(u8) -> bool) -> bool {
654 let mut seen = false;
655 while matches!(self.peek(0), Some(c) if is_digit(c) || c == b'_') {
656 seen |= self.peek(0) != Some(b'_');
657 self.pos += 1;
658 }
659 seen
660 }
661
662 fn lex_ident_or_keyword(&mut self, start: usize) {
663 self.scan_ident();
664 let text = &self.input[start..self.pos];
665 let keyword = keyword_kind(text);
666 if keyword.is_none()
671 && match self.peek(0) {
672 Some(b'"') => text != "typegroup",
673 Some(b'`') => true,
674 _ => false,
675 }
676 {
677 self.push(TokKind::StringPrefix, start, self.pos);
678 let open = self.pos;
679 if self.peek(0) == Some(b'"') {
680 self.lex_open_string(open, true);
681 } else {
682 self.lex_open_cmd(open, true);
683 }
684 return;
685 }
686 self.push(keyword.unwrap_or(TokKind::Ident), start, self.pos);
687 }
688
689 fn lex_interp_ident(&mut self) {
693 let start = self.pos;
694 self.scan_ident();
695 self.push(TokKind::Ident, start, self.pos);
696 }
697
698 fn scan_ident(&mut self) {
701 self.pos += self.char_at(self.pos).len_utf8();
702 loop {
703 let c = self.char_at(self.pos);
704 if c == '!' && self.peek(1) == Some(b'=') {
710 break;
711 }
712 if self.pos < self.bytes.len() && is_ident_continue(c) {
713 self.pos += c.len_utf8();
714 } else {
715 break;
716 }
717 }
718 }
719}
720
721#[rustfmt::skip]
731const OPS: &[(&[u8], TokKind)] = &[
732 (b".>>>=", TokKind::DotUShrEq),
737 (b".<-->", TokKind::DotLeftRightArrow),
738 (b".>>>", TokKind::DotUShr),
739 (b".//=", TokKind::DotSlashSlashEq),
740 (b".<<=", TokKind::DotShlEq),
741 (b".>>=", TokKind::DotShrEq),
742 (b".-->", TokKind::DotLongArrow),
743 (b".<--", TokKind::DotLeftLongArrow),
744 (b".===", TokKind::DotEqEqEq),
745 (b".!==", TokKind::DotNotEqEq),
746 (b".+%=", TokKind::DotPlusPercentEq),
747 (b".-%=", TokKind::DotMinusPercentEq),
748 (b".*%=", TokKind::DotStarPercentEq),
749 (b"...", TokKind::DotDotDot),
750 (b".==", TokKind::DotEqEq),
751 (b".!=", TokKind::DotNotEq),
752 (b".<=", TokKind::DotLe),
753 (b".>=", TokKind::DotGe),
754 (b".<<", TokKind::DotShl),
755 (b".>>", TokKind::DotShr),
756 (b".<:", TokKind::DotSubtype),
757 (b".>:", TokKind::DotSupertype),
758 (b".//", TokKind::DotSlashSlash),
759 (b".**", TokKind::DotStarStar),
761 (b".--", TokKind::DotMinusMinus),
762 (b".=>", TokKind::DotFatArrow),
763 (b".|>", TokKind::DotPipeGt),
764 (b".&&", TokKind::DotAndAnd),
765 (b".||", TokKind::DotOrOr),
766 (b".+%", TokKind::DotPlusPercent),
767 (b".-%", TokKind::DotMinusPercent),
768 (b".*%", TokKind::DotStarPercent),
769 (b".+=", TokKind::DotPlusEq),
771 (b".-=", TokKind::DotMinusEq),
772 (b".*=", TokKind::DotStarEq),
773 (b"./=", TokKind::DotSlashEq),
774 (b".\\=", TokKind::DotBackslashEq),
775 (b".^=", TokKind::DotCaretEq),
776 (b".%=", TokKind::DotPercentEq),
777 (b".&=", TokKind::DotAmpEq),
778 (b".|=", TokKind::DotPipeEq),
779 (b"..", TokKind::DotDot),
780 (b".+", TokKind::DotPlus),
781 (b".-", TokKind::DotMinus),
782 (b".*", TokKind::DotStar),
783 (b"./", TokKind::DotSlash),
784 (b".\\", TokKind::DotBackslash),
785 (b".^", TokKind::DotCaret),
786 (b".%", TokKind::DotPercent),
787 (b".=", TokKind::DotEq),
788 (b".<", TokKind::DotLt),
789 (b".>", TokKind::DotGt),
790 (b".~", TokKind::DotTilde),
791 (b".&", TokKind::DotAmp),
792 (b".|", TokKind::DotPipe),
793 (b".!", TokKind::DotBang),
795 (b".", TokKind::Dot),
796 (b"<-->", TokKind::LeftRightArrow),
798 (b"<--", TokKind::LeftLongArrow),
799 (b"<<=", TokKind::ShlEq),
800 (b"<=", TokKind::Le),
801 (b"<:", TokKind::Subtype),
802 (b"<|", TokKind::PipeLt),
803 (b"<<", TokKind::Shl),
804 (b"<", TokKind::Lt),
805 (b">>>=", TokKind::UShrEq),
807 (b">>>", TokKind::UShr),
808 (b">>=", TokKind::ShrEq),
809 (b">=", TokKind::Ge),
810 (b">:", TokKind::Supertype),
811 (b">>", TokKind::Shr),
812 (b">", TokKind::Gt),
813 (b"===", TokKind::EqEqEq),
815 (b"==", TokKind::EqEq),
816 (b"=>", TokKind::FatArrow),
817 (b"=", TokKind::Eq),
818 (b"!==", TokKind::NotEqEq),
821 (b"!=", TokKind::NotEq),
822 (b"!", TokKind::Bang),
823 (b"+%=", TokKind::PlusPercentEq),
826 (b"+%", TokKind::PlusPercent),
827 (b"++", TokKind::PlusPlus),
828 (b"+=", TokKind::PlusEq),
829 (b"+", TokKind::Plus),
830 (b"-->", TokKind::LongArrow),
832 (b"-%=", TokKind::MinusPercentEq),
833 (b"--", TokKind::MinusMinus),
834 (b"->", TokKind::Arrow),
835 (b"-%", TokKind::MinusPercent),
836 (b"-=", TokKind::MinusEq),
837 (b"-", TokKind::Minus),
838 (b"*%=", TokKind::StarPercentEq),
840 (b"**", TokKind::StarStar),
841 (b"*%", TokKind::StarPercent),
842 (b"*=", TokKind::StarEq),
843 (b"*", TokKind::Star),
844 (b"//=", TokKind::SlashSlashEq),
846 (b"//", TokKind::SlashSlash),
847 (b"/=", TokKind::SlashEq),
848 (b"/", TokKind::Slash),
849 (b"\\=", TokKind::BackslashEq),
851 (b"\\", TokKind::Backslash),
852 (b"^=", TokKind::CaretEq),
854 (b"^", TokKind::Caret),
855 (b"%=", TokKind::PercentEq),
857 (b"%", TokKind::Percent),
858 (b"&&", TokKind::AndAnd),
860 (b"&=", TokKind::AmpEq),
861 (b"&", TokKind::Amp),
862 (b"||", TokKind::OrOr),
864 (b"|>", TokKind::PipeGt),
865 (b"|=", TokKind::PipeEq),
866 (b"|", TokKind::Pipe),
867 (b"::", TokKind::ColonColon),
869 (b":=", TokKind::ColonEq),
870 (b":", TokKind::Colon),
871 (b"$=", TokKind::DollarEq),
874 (b"$", TokKind::Dollar),
875 (b"~", TokKind::Tilde),
877 (b"?", TokKind::Question),
878 (b"@", TokKind::At),
879 (b"(", TokKind::LParen),
880 (b")", TokKind::RParen),
881 (b"[", TokKind::LBracket),
882 (b"]", TokKind::RBracket),
883 (b"{", TokKind::LBrace),
884 (b"}", TokKind::RBrace),
885 (b",", TokKind::Comma),
886 (b";", TokKind::Semicolon),
887];
888
889#[derive(Clone, Copy)]
891struct OpGroup {
892 start: u8,
895 end: u8,
896 seconds: u128,
902}
903
904impl OpGroup {
905 fn extends(&self, next: Option<&u8>) -> bool {
908 matches!(next, Some(&b) if b < 128 && self.seconds >> b & 1 == 1)
909 }
910}
911
912const OP_INDEX: [OpGroup; 256] = build_op_index();
914
915const fn build_op_index() -> [OpGroup; 256] {
922 assert!(OPS.len() < u8::MAX as usize, "OPS outgrew its u8 index");
923 let mut index = [OpGroup {
924 start: 0,
925 end: 0,
926 seconds: 0,
927 }; 256];
928 let mut i = 0;
929 while i < OPS.len() {
930 let first = OPS[i].0[0];
931 assert!(
932 index[first as usize].start == index[first as usize].end,
933 "OPS entries sharing a first byte must be contiguous"
934 );
935 let start = i;
936 let mut len = usize::MAX;
937 let mut seconds = 0u128;
938 while i < OPS.len() && OPS[i].0[0] == first {
939 let text = OPS[i].0;
940 assert!(
941 text.len() <= len,
942 "OPS entries must run longest-first within a first-byte group"
943 );
944 len = text.len();
945 if len > 1 {
946 assert!(
947 text[1] < 128,
948 "OPS holds ASCII spellings only; a Unicode operator belongs in `unicode_op_at`"
949 );
950 seconds |= 1u128 << text[1];
951 }
952 i += 1;
953 }
954 assert!(
955 len == 1,
956 "every OPS group must end with its own single-byte spelling"
957 );
958 index[first as usize] = OpGroup {
959 start: start as u8,
960 end: i as u8,
961 seconds,
962 };
963 }
964 index
965}
966
967fn try_ascii_op(rest: &[u8]) -> Option<(TokKind, usize)> {
972 let group = &OP_INDEX[*rest.first()? as usize];
973 let entries = &OPS[group.start as usize..group.end as usize];
974 if !group.extends(rest.get(1)) {
977 let (text, kind) = *entries.last()?;
978 return Some((kind, text.len()));
979 }
980 let next = rest[1]; entries
982 .iter()
983 .find(|(text, _)| match text.get(1) {
984 Some(&second) => second == next && rest.starts_with(text),
988 None => true,
989 })
990 .map(|&(text, kind)| (kind, text.len()))
991}
992
993macro_rules! define_keyword_tables {
997 ($($text:literal $tok:ident $syn:ident,)*) => {
998 pub const KEYWORDS: &[&str] = &[$($text),*];
1002
1003 fn keyword_kind(text: &str) -> Option<TokKind> {
1006 Some(match text {
1007 $($text => TokKind::$tok,)*
1008 _ => return None,
1009 })
1010 }
1011
1012 impl TokKind {
1013 pub(crate) fn is_keyword(self) -> bool {
1016 matches!(self, $(TokKind::$tok)|*)
1017 }
1018 }
1019 };
1020}
1021
1022keyword_table!(define_keyword_tables);
1023
1024pub fn is_ident_start(c: char) -> bool {
1033 if c.is_ascii() {
1034 c == '_' || c.is_ascii_alphabetic()
1035 } else {
1036 c != '\u{1f8b2}' && super::unicode_ident::is_unicode_ident_start(c)
1037 }
1038}
1039
1040pub fn is_ident_continue(c: char) -> bool {
1045 if c.is_ascii() {
1046 c == '_' || c == '!' || c.is_ascii_alphanumeric()
1047 } else {
1048 super::unicode_ident::is_unicode_ident_continue(c)
1049 }
1050}
1051
1052const OP_SUFFIX_CHARS: &str = "²³¹ʰʲʳʷʸˡˢˣ˱˲ᴬᴮᴰᴱᴳᴴᴵᴶᴷᴸᴹᴺᴼᴾᴿᵀᵁᵂᵃᵅᵇᵈᵉᵋᵍᵏᵐᵒᵖᵗᵘᵛᵝᵞᵟᵠᵡᵢᵣᵤᵥᵦᵧᵨᵩᵪᶜᶠᶥᶦᶫᶰᶲᶸᶻᶿ′″‴‵‶‷⁗⁰ⁱ⁴⁵⁶⁷⁸⁹⁺⁻⁼⁽⁾ⁿ₀₁₂₃₄₅₆₇₈₉₊₋₌₍₎ₐₑₒₓₔₕₖₗₘₙₚₛₜⱼⱽꜛꜜꜝ";
1057
1058pub(crate) fn is_op_suffix_char(c: char) -> bool {
1061 OP_SUFFIX_CHARS.contains(c)
1062}
1063
1064fn op_takes_suffix(kind: TokKind) -> bool {
1070 use TokKind::*;
1071 matches!(
1072 kind,
1073 Plus | Minus
1074 | Star
1075 | Slash
1076 | Backslash
1077 | SlashSlash
1078 | Caret
1079 | Percent
1080 | PlusPercent
1081 | MinusPercent
1082 | StarPercent
1083 | EqEq
1084 | NotEq
1085 | EqEqEq
1086 | NotEqEq
1087 | Lt
1088 | Le
1089 | Gt
1090 | Ge
1091 | Amp
1092 | Pipe
1093 | Shl
1094 | Shr
1095 | UShr
1096 | PipeGt
1097 | PipeLt
1098 | FatArrow
1099 | LongArrow
1100 | LeftRightArrow
1101 | LeftLongArrow
1102 | Transpose
1103 | DotPlus
1104 | DotMinus
1105 | DotStar
1106 | DotPlusPercent
1107 | DotMinusPercent
1108 | DotStarPercent
1109 | DotSlash
1110 | DotBackslash
1111 | DotSlashSlash
1112 | DotCaret
1113 | DotPercent
1114 | DotEqEq
1115 | DotNotEq
1116 | DotEqEqEq
1117 | DotNotEqEq
1118 | DotLt
1119 | DotLe
1120 | DotGt
1121 | DotGe
1122 | DotShl
1123 | DotShr
1124 | DotUShr
1125 | DotFatArrow
1126 | DotLongArrow
1127 | DotLeftLongArrow
1128 | DotLeftRightArrow
1129 | DotPipeGt
1130 | DotAmp
1131 | DotPipe
1132 | UniArrow
1133 | UniComparison
1134 | UniColon
1135 | UniPlus
1136 | UniTimes
1137 | UniPower
1138 )
1139}
1140
1141const MINUS_SIGN: char = '\u{2212}';
1143
1144const DIVIDE_SIGN: char = '\u{f7}';
1147const XOR_SIGN: char = '\u{22bb}';
1148
1149fn is_unicode_infix_tier(kind: TokKind) -> bool {
1153 use TokKind::*;
1154 matches!(
1155 kind,
1156 UniArrow | UniComparison | UniColon | UniPlus | UniTimes | UniPower
1157 )
1158}
1159
1160#[cfg(test)]
1161mod tests {
1162 use super::*;
1163
1164 fn kinds(input: &str) -> Vec<TokKind> {
1165 lex(input).into_iter().map(|t| t.kind).collect()
1166 }
1167
1168 fn roundtrips(input: &str) -> bool {
1169 let joined: String = lex(input).into_iter().map(|t| t.text).collect();
1170 joined == input
1171 }
1172
1173 #[test]
1174 fn every_keyword_lexes_as_one_keyword_token() {
1175 for kw in KEYWORDS {
1181 assert_eq!(kinds(kw), vec![keyword_kind(kw).unwrap()], "lexing {kw}");
1182 assert!(keyword_kind(kw).unwrap().is_keyword(), "{kw} is a keyword");
1183 assert!(
1184 crate::parser::tree_builder::syntax_kind_for(keyword_kind(kw).unwrap())
1185 .is_keyword(),
1186 "{kw} materializes as a keyword kind"
1187 );
1188 }
1189 }
1190
1191 #[test]
1195 fn the_wide_arrow_operator_is_not_an_identifier_start() {
1196 assert_eq!(
1197 kinds("x \u{1f8b2} y"),
1198 vec![
1199 TokKind::Ident,
1200 TokKind::Whitespace,
1201 TokKind::UniArrow,
1202 TokKind::Whitespace,
1203 TokKind::Ident,
1204 ]
1205 );
1206 assert_eq!(kinds("a\u{1f8b2}b"), vec![TokKind::Ident]);
1209 }
1210
1211 #[test]
1212 fn lossless_over_assorted_input() {
1213 for input in [
1214 "x = 1 + 2\n",
1215 "f(a, b)\n",
1216 "#= a #= nested =# b =#\n",
1217 "function g(x)\n x ^ 2\nend\n",
1218 "s = \"hello\\n\"\nc = 'a'\n",
1219 "if a >= b\n a\nelseif c\n c\nelse\n b\nend\n",
1220 "for i in 1:10\n x\nend\n",
1221 "while x > 0\n x\nend\n",
1222 "map(xs) do x, y\n x + y\nend\n",
1223 "try\n f()\ncatch e\n g()\nfinally\n h()\nend\n",
1224 "mutable struct Counter\n n\nend\n",
1225 "module M\nend\n",
1226 "α = β + 1\n",
1227 "0x1f + 0b1010\n",
1228 "x = 0o755\ny = 0x1.8p3\nz = 1f0\n",
1229 "r = 3//4 + 1_000\nq = a .// b\n",
1230 "n = 1.5e-3\nm = .5\nk = 2.\n",
1231 "s = \"a$x b\"\n",
1232 "s = \"a$(f(x))b\"\n",
1233 "s = \"\"\"x$(y)\"\"\"\n",
1234 "c = `echo $x`\n",
1235 "r = raw\"\\d+\"\n",
1236 "m = r\"pat\"ims\n",
1237 "v = v\"1.2.3\"\n",
1238 "b = b\"\\x00\"\n",
1239 "s = \"$foo.bar\"\n",
1240 "s = \"\\$lit\"\n",
1241 "s = \"$$\"\n",
1242 "s = \"unterminated\n",
1243 "s = \"$(g(\"nested\"))\"\n",
1244 "s = \"a\\αb\"\n",
1247 "s = \"\\α",
1248 ] {
1249 assert!(roundtrips(input), "did not round-trip: {input:?}");
1250 }
1251 }
1252
1253 #[test]
1254 fn interpolation_with_bare_ident() {
1255 assert_eq!(
1256 kinds("\"a$x\""),
1257 vec![
1258 TokKind::StringDelimOpen,
1259 TokKind::StringContent,
1260 TokKind::Dollar,
1261 TokKind::Ident,
1262 TokKind::StringDelimClose,
1263 ]
1264 );
1265 }
1266
1267 #[test]
1268 fn interpolation_with_parenthesized_expr() {
1269 assert_eq!(
1270 kinds("\"$(y)\""),
1271 vec![
1272 TokKind::StringDelimOpen,
1273 TokKind::Dollar,
1274 TokKind::LParen,
1275 TokKind::Ident,
1276 TokKind::RParen,
1277 TokKind::StringDelimClose,
1278 ]
1279 );
1280 }
1281
1282 #[test]
1283 fn nested_parens_in_interpolation() {
1284 assert_eq!(
1286 kinds("\"$(f(x))\""),
1287 vec![
1288 TokKind::StringDelimOpen,
1289 TokKind::Dollar,
1290 TokKind::LParen,
1291 TokKind::Ident,
1292 TokKind::LParen,
1293 TokKind::Ident,
1294 TokKind::RParen,
1295 TokKind::RParen,
1296 TokKind::StringDelimClose,
1297 ]
1298 );
1299 }
1300
1301 #[test]
1302 fn raw_literal_does_not_interpolate() {
1303 assert_eq!(
1305 kinds("raw\"$x\\d\""),
1306 vec![
1307 TokKind::StringPrefix,
1308 TokKind::StringDelimOpen,
1309 TokKind::StringContent,
1310 TokKind::StringDelimClose,
1311 ]
1312 );
1313 }
1314
1315 #[test]
1316 fn prefix_and_suffix_flags() {
1317 assert_eq!(
1318 kinds("r\"pat\"ims"),
1319 vec![
1320 TokKind::StringPrefix,
1321 TokKind::StringDelimOpen,
1322 TokKind::StringContent,
1323 TokKind::StringDelimClose,
1324 TokKind::StringSuffix,
1325 ]
1326 );
1327 }
1328
1329 #[test]
1330 fn command_literal_interpolates() {
1331 assert_eq!(
1332 kinds("`cmd $x`"),
1333 vec![
1334 TokKind::CmdDelimOpen,
1335 TokKind::StringContent,
1336 TokKind::Dollar,
1337 TokKind::Ident,
1338 TokKind::CmdDelimClose,
1339 ]
1340 );
1341 }
1342
1343 #[test]
1344 fn escaped_dollar_is_content() {
1345 assert_eq!(
1347 kinds("\"\\$x\""),
1348 vec![
1349 TokKind::StringDelimOpen,
1350 TokKind::StringContent,
1351 TokKind::StringDelimClose,
1352 ]
1353 );
1354 }
1355
1356 #[test]
1357 fn plain_string_adjacent_ident_is_not_a_suffix() {
1358 assert_eq!(
1360 kinds("\"a\"b"),
1361 vec![
1362 TokKind::StringDelimOpen,
1363 TokKind::StringContent,
1364 TokKind::StringDelimClose,
1365 TokKind::Ident,
1366 ]
1367 );
1368 }
1369
1370 #[test]
1371 fn prefix_requires_adjacent_quote() {
1372 assert_eq!(
1374 kinds("r \"x\""),
1375 vec![
1376 TokKind::Ident,
1377 TokKind::Whitespace,
1378 TokKind::StringDelimOpen,
1379 TokKind::StringContent,
1380 TokKind::StringDelimClose,
1381 ]
1382 );
1383 }
1384
1385 #[test]
1386 fn keywords_and_operators() {
1387 assert_eq!(
1388 kinds("a == b"),
1389 vec![
1390 TokKind::Ident,
1391 TokKind::Whitespace,
1392 TokKind::EqEq,
1393 TokKind::Whitespace,
1394 TokKind::Ident
1395 ]
1396 );
1397 assert_eq!(keyword_kind("function"), Some(TokKind::FunctionKw));
1398 assert_eq!(keyword_kind("ends"), None);
1399 }
1400
1401 #[test]
1402 fn bang_in_identifier() {
1403 assert_eq!(kinds("push!"), vec![TokKind::Ident]);
1404 assert_eq!(kinds("a!b"), vec![TokKind::Ident]);
1407 assert_eq!(
1408 kinds("a!=b"),
1409 vec![TokKind::Ident, TokKind::NotEq, TokKind::Ident]
1410 );
1411 assert_eq!(
1412 kinds("a!==b"),
1413 vec![TokKind::Ident, TokKind::NotEqEq, TokKind::Ident]
1414 );
1415 assert_eq!(
1417 kinds("a!!=b"),
1418 vec![TokKind::Ident, TokKind::NotEq, TokKind::Ident]
1419 );
1420 }
1421
1422 #[test]
1423 fn unicode_identifier_chars() {
1424 assert_eq!(kinds("x\u{304}"), vec![TokKind::Ident]);
1426 assert_eq!(kinds("x\u{302}r"), vec![TokKind::Ident]);
1427 assert_eq!(kinds("\u{3b1}\u{2032}"), vec![TokKind::Ident]);
1429 assert_eq!(kinds("\u{3c1}\u{221e}"), vec![TokKind::Ident]);
1430 assert_eq!(kinds("\u{2207}batchnorm"), vec![TokKind::Ident]);
1432 assert_eq!(kinds("\u{2081}"), vec![TokKind::ErrorIdentifierStart]);
1435 assert!(roundtrips("x\u{304} = 1"));
1437 assert!(roundtrips("\u{2207}f(x) = x"));
1438 }
1439
1440 #[test]
1441 fn unicode_17_identifier_chars() {
1442 assert_eq!(kinds("\u{88f}"), vec![TokKind::Ident]);
1443 assert_eq!(kinds("\u{11db0}"), vec![TokKind::Ident]);
1444 assert_eq!(kinds("x\u{11b60}"), vec![TokKind::Ident]);
1445 assert_eq!(kinds("\u{11b60}"), vec![TokKind::ErrorIdentifierStart]);
1446 assert!(roundtrips("\u{88f} = \u{11db0} + x\u{11b60}"));
1447 }
1448
1449 #[test]
1450 fn unicode_18_identifier_chars() {
1451 for c in [
1452 '\u{558}',
1453 '\u{209d}',
1454 '\u{209e}',
1455 '\u{209f}',
1456 '\u{a7dd}',
1457 '\u{107bb}',
1458 '\u{10ed9}',
1459 '\u{12550}',
1460 '\u{18e00}',
1461 '\u{1dfd0}',
1462 '\u{3d000}',
1463 '\u{3fc3f}',
1464 ] {
1465 for input in [c.to_string(), format!("x{c}!")] {
1466 assert_eq!(kinds(&input), vec![TokKind::Ident], "{input:?}");
1467 assert!(roundtrips(&input));
1468 }
1469 }
1470 for c in [
1471 '\u{5c8}',
1472 '\u{b53}',
1473 '\u{1ade}',
1474 '\u{10ec9}',
1475 '\u{11df0}',
1476 '\u{1d127}',
1477 '\u{1d250}',
1478 ] {
1479 assert_eq!(kinds(&format!("x{c}")), vec![TokKind::Ident], "{c:?}");
1480 assert_eq!(
1481 kinds(&c.to_string()),
1482 vec![TokKind::ErrorIdentifierStart],
1483 "{c:?}"
1484 );
1485 assert!(roundtrips(&format!("{c} x{c}")));
1486 }
1487 for c in ['\u{3cfff}', '\u{3fc40}'] {
1488 assert_eq!(kinds(&c.to_string()), vec![TokKind::Unknown], "{c:?}");
1489 }
1490 }
1491
1492 #[test]
1493 fn unicode_18_operator_suffixes() {
1494 for suffix in "ᵅᵋᶲ˱˲ₔ\u{1dfd0}".chars() {
1495 for (op, kind) in [
1496 ("+", TokKind::Plus),
1497 (".+", TokKind::DotPlus),
1498 ("⊗", TokKind::UniTimes),
1499 (".⊗", TokKind::UniTimes),
1500 ("-->", TokKind::LongArrow),
1501 ("'", TokKind::Transpose),
1502 ] {
1503 let operator = format!("{op}{suffix}");
1504 let input = format!("a{operator}b");
1505 let tokens = lex(&input);
1506 assert_eq!(tokens.len(), 3, "{input:?}: {tokens:?}");
1507 assert_eq!(tokens[1].kind, kind, "{input:?}");
1508 assert_eq!(tokens[1].text, operator, "{input:?}");
1509 assert_eq!(tokens[1].start, 1);
1510 assert_eq!(tokens[1].end, 1 + operator.len());
1511 assert_eq!(tokens[2].text, "b");
1512 assert!(roundtrips(&input));
1513 }
1514 for op in ["=", ":", "::", "&&", "||", "->", "<:", "!", "~", "+="] {
1515 let input = format!("{op}{suffix}");
1516 assert_eq!(lex(&input)[0].text, op, "{input:?}");
1517 assert!(roundtrips(&input));
1518 }
1519 }
1520 }
1521
1522 #[test]
1523 fn identity_operators() {
1524 assert_eq!(
1526 kinds("a===b"),
1527 vec![TokKind::Ident, TokKind::EqEqEq, TokKind::Ident]
1528 );
1529 assert_eq!(
1530 kinds("a==b"),
1531 vec![TokKind::Ident, TokKind::EqEq, TokKind::Ident]
1532 );
1533 }
1534
1535 #[test]
1536 fn numeric_literal_kinds() {
1537 assert_eq!(kinds("123"), vec![TokKind::Integer]);
1539 assert_eq!(kinds("1_000"), vec![TokKind::Integer]);
1540 assert_eq!(
1541 kinds("12345678901234567890123456789"),
1542 vec![TokKind::Integer]
1543 );
1544 assert_eq!(kinds("0x1f"), vec![TokKind::HexInt]);
1546 assert_eq!(kinds("0o755"), vec![TokKind::OctInt]);
1547 assert_eq!(kinds("0b1010"), vec![TokKind::BinInt]);
1548 assert_eq!(kinds("3.14"), vec![TokKind::Float]);
1550 assert_eq!(kinds(".5"), vec![TokKind::Float]);
1551 assert_eq!(kinds("2."), vec![TokKind::Float]);
1552 assert_eq!(kinds("1.5e-3"), vec![TokKind::Float]);
1553 assert_eq!(kinds("1f0"), vec![TokKind::Float32]);
1555 assert_eq!(kinds("2.5f-3"), vec![TokKind::Float32]);
1556 assert_eq!(kinds("0x1p0"), vec![TokKind::Float]);
1557 assert_eq!(kinds("0x1.8p3"), vec![TokKind::Float]);
1558 assert_eq!(kinds("0x1.p3"), vec![TokKind::Float]);
1561 assert_eq!(kinds("0x.8p3"), vec![TokKind::Float]);
1562 assert_eq!(kinds("0x1P+2"), vec![TokKind::Float]);
1563 assert_eq!(kinds("0x1_2p3"), vec![TokKind::Float]);
1564 assert_eq!(kinds("0x1e3"), vec![TokKind::HexInt]);
1566 }
1567
1568 #[test]
1569 fn malformed_hex_literal_kinds() {
1570 for src in [
1574 "0x1p", "0x1p+", "0x1.8p", "0x1.p", "0x.p3", "0xp3", "0x", "0x_",
1575 ] {
1576 assert_eq!(
1577 kinds(src),
1578 vec![TokKind::ErrorInvalidNumber],
1579 "{src} should be an invalid numeric constant"
1580 );
1581 }
1582 for src in ["0x1.8", "0x1.", "0x.8", "0x.", "0x1.8e2"] {
1584 assert_eq!(
1585 kinds(src),
1586 vec![TokKind::ErrorHexFloatNoP],
1587 "{src} should require a p exponent"
1588 );
1589 }
1590 }
1591
1592 #[test]
1593 fn rational_operators() {
1594 assert_eq!(
1595 kinds("3//4"),
1596 vec![TokKind::Integer, TokKind::SlashSlash, TokKind::Integer]
1597 );
1598 assert_eq!(
1599 kinds("a .// b"),
1600 vec![
1601 TokKind::Ident,
1602 TokKind::Whitespace,
1603 TokKind::DotSlashSlash,
1604 TokKind::Whitespace,
1605 TokKind::Ident,
1606 ]
1607 );
1608 }
1609
1610 #[test]
1611 fn left_division_operators() {
1612 assert_eq!(
1616 kinds("a\\b"),
1617 vec![TokKind::Ident, TokKind::Backslash, TokKind::Ident]
1618 );
1619 assert_eq!(
1620 kinds("a\\=b"),
1621 vec![TokKind::Ident, TokKind::BackslashEq, TokKind::Ident]
1622 );
1623 assert_eq!(
1624 kinds("a .\\ b"),
1625 vec![
1626 TokKind::Ident,
1627 TokKind::Whitespace,
1628 TokKind::DotBackslash,
1629 TokKind::Whitespace,
1630 TokKind::Ident,
1631 ]
1632 );
1633 assert_eq!(
1634 kinds("a.\\=b"),
1635 vec![TokKind::Ident, TokKind::DotBackslashEq, TokKind::Ident]
1636 );
1637 }
1638
1639 #[test]
1640 fn left_arrow_operators() {
1641 assert_eq!(
1645 kinds("a<--b"),
1646 vec![TokKind::Ident, TokKind::LeftLongArrow, TokKind::Ident]
1647 );
1648 assert_eq!(
1649 kinds("a<-->b"),
1650 vec![TokKind::Ident, TokKind::LeftRightArrow, TokKind::Ident]
1651 );
1652 assert_eq!(
1653 kinds("a.<--b"),
1654 vec![TokKind::Ident, TokKind::DotLeftLongArrow, TokKind::Ident]
1655 );
1656 assert_eq!(
1657 kinds("a.<-->b"),
1658 vec![TokKind::Ident, TokKind::DotLeftRightArrow, TokKind::Ident]
1659 );
1660 assert_eq!(
1661 kinds("a<-b"),
1662 vec![TokKind::Ident, TokKind::Lt, TokKind::Minus, TokKind::Ident]
1663 );
1664 }
1665
1666 #[test]
1667 fn compound_shift_and_unicode_assignments() {
1668 assert_eq!(
1671 kinds("a<<=b"),
1672 vec![TokKind::Ident, TokKind::ShlEq, TokKind::Ident]
1673 );
1674 assert_eq!(
1675 kinds("a>>=b"),
1676 vec![TokKind::Ident, TokKind::ShrEq, TokKind::Ident]
1677 );
1678 assert_eq!(
1679 kinds("a>>>=b"),
1680 vec![TokKind::Ident, TokKind::UShrEq, TokKind::Ident]
1681 );
1682 assert_eq!(
1684 kinds("a÷=b"),
1685 vec![TokKind::Ident, TokKind::DivEq, TokKind::Ident]
1686 );
1687 assert_eq!(
1688 kinds("a⊻=b"),
1689 vec![TokKind::Ident, TokKind::XorEq, TokKind::Ident]
1690 );
1691 assert_eq!(
1693 kinds("a.<<=b"),
1694 vec![TokKind::Ident, TokKind::DotShlEq, TokKind::Ident]
1695 );
1696 assert_eq!(
1697 kinds("a.>>>=b"),
1698 vec![TokKind::Ident, TokKind::DotUShrEq, TokKind::Ident]
1699 );
1700 assert_eq!(
1701 kinds("a.÷=b"),
1702 vec![TokKind::Ident, TokKind::DotDivEq, TokKind::Ident]
1703 );
1704 assert_eq!(
1705 kinds("a.⊻=b"),
1706 vec![TokKind::Ident, TokKind::DotXorEq, TokKind::Ident]
1707 );
1708 }
1709
1710 #[test]
1711 fn uppercase_hex_prefix_is_not_a_literal() {
1712 assert_eq!(kinds("0X1"), vec![TokKind::Integer, TokKind::Ident]);
1714 }
1715
1716 #[test]
1717 fn inf_and_nan_are_identifiers() {
1718 assert_eq!(kinds("Inf"), vec![TokKind::Ident]);
1720 assert_eq!(kinds("NaN"), vec![TokKind::Ident]);
1721 assert_eq!(kinds("Inf32"), vec![TokKind::Ident]);
1722 }
1723
1724 #[test]
1725 fn subtype_and_supertype_operators() {
1726 assert_eq!(
1727 kinds("T<:U"),
1728 vec![TokKind::Ident, TokKind::Subtype, TokKind::Ident]
1729 );
1730 assert_eq!(
1731 kinds("T>:U"),
1732 vec![TokKind::Ident, TokKind::Supertype, TokKind::Ident]
1733 );
1734 }
1735
1736 #[test]
1737 fn splat_is_three_dots() {
1738 assert_eq!(kinds("x..."), vec![TokKind::Ident, TokKind::DotDotDot]);
1739 assert_eq!(kinds(".."), vec![TokKind::DotDot]);
1741 assert_eq!(
1742 kinds("a.b"),
1743 vec![TokKind::Ident, TokKind::Dot, TokKind::Ident]
1744 );
1745 }
1746
1747 #[test]
1748 fn broadcasting_operators() {
1749 assert_eq!(
1750 kinds("a .+ b"),
1751 vec![
1752 TokKind::Ident,
1753 TokKind::Whitespace,
1754 TokKind::DotPlus,
1755 TokKind::Whitespace,
1756 TokKind::Ident
1757 ]
1758 );
1759 assert_eq!(kinds("x .== y").get(2), Some(&TokKind::DotEqEq));
1761 assert_eq!(kinds("x .= y").get(2), Some(&TokKind::DotEq));
1762 assert_eq!(kinds("a .<= b").get(2), Some(&TokKind::DotLe));
1763 assert_eq!(kinds("x .=== y").get(2), Some(&TokKind::DotEqEqEq));
1765 assert_eq!(kinds("x .!== y").get(2), Some(&TokKind::DotNotEqEq));
1766 assert_eq!(kinds(".!y").first(), Some(&TokKind::DotBang));
1769 assert_eq!(kinds("x .!= y").get(2), Some(&TokKind::DotNotEq));
1770 assert_eq!(kinds(".√[3]").first(), Some(&TokKind::UniRadical));
1773 assert_eq!(kinds(".¬a").first(), Some(&TokKind::UniRadical));
1774 assert_eq!(kinds("a .&= b").get(2), Some(&TokKind::DotAmpEq));
1777 assert_eq!(kinds("a .|= b").get(2), Some(&TokKind::DotPipeEq));
1778 assert_eq!(kinds("a .& b").get(2), Some(&TokKind::DotAmp));
1779 assert_eq!(kinds("a .| b").get(2), Some(&TokKind::DotPipe));
1780 assert_eq!(kinds("a .|> b").get(2), Some(&TokKind::DotPipeGt));
1781 assert_eq!(
1783 kinds("a.b"),
1784 vec![TokKind::Ident, TokKind::Dot, TokKind::Ident]
1785 );
1786 assert_eq!(kinds(".."), vec![TokKind::DotDot]);
1788 assert_eq!(
1790 kinds("f.(x)"),
1791 vec![
1792 TokKind::Ident,
1793 TokKind::Dot,
1794 TokKind::LParen,
1795 TokKind::Ident,
1796 TokKind::RParen
1797 ]
1798 );
1799 }
1800
1801 #[test]
1802 fn where_is_a_keyword() {
1803 assert_eq!(keyword_kind("where"), Some(TokKind::WhereKw));
1804 assert_eq!(kinds("where"), vec![TokKind::WhereKw]);
1805 }
1806
1807 #[test]
1808 fn every_ops_entry_lexes_as_itself() {
1809 for &(text, kind) in OPS {
1814 let spelling = std::str::from_utf8(text).expect("OPS spellings are ASCII");
1815 let tokens = lex(spelling);
1816 assert_eq!(
1817 tokens.iter().map(|t| t.kind).collect::<Vec<_>>(),
1818 vec![kind],
1819 "{spelling:?} did not lex as one {kind:?}"
1820 );
1821 assert_eq!(tokens[0].text, spelling);
1822 }
1823 }
1824
1825 #[test]
1826 fn longest_match_beats_every_shared_prefix() {
1827 for spelling in [
1830 ".>>>=", ".<-->", ".<--", ".//=", ".===", ".!==", "...", "<-->", ">>>=", "-->", "//=",
1831 "+%=", ".+%", ".-%", ".*%", ".+%=", ".-%=", ".*%=", "!==", "===",
1832 ] {
1833 assert_eq!(kinds(spelling).len(), 1, "{spelling:?} was split");
1834 }
1835 assert_eq!(kinds("<-"), vec![TokKind::Lt, TokKind::Minus]);
1838 assert_eq!(kinds("+++"), vec![TokKind::PlusPlus, TokKind::Plus]);
1839 assert_eq!(kinds("++="), vec![TokKind::PlusPlus, TokKind::Eq]);
1840 assert_eq!(
1841 kinds("a.b"),
1842 vec![TokKind::Ident, TokKind::Dot, TokKind::Ident]
1843 );
1844 }
1845
1846 #[test]
1847 fn unicode_operators_beat_the_ascii_table() {
1848 assert_eq!(kinds(".×"), vec![TokKind::UniTimes]);
1851 assert_eq!(kinds(".√"), vec![TokKind::UniRadical]);
1852 assert_eq!(kinds(".÷="), vec![TokKind::DotDivEq]);
1853 assert_eq!(kinds(".⊻="), vec![TokKind::DotXorEq]);
1854 assert_eq!(kinds("−"), vec![TokKind::Minus]);
1856 assert_eq!(kinds("−="), vec![TokKind::MinusEq]);
1857 assert_eq!(kinds(".−="), vec![TokKind::DotMinusEq]);
1858 assert_eq!(kinds("÷="), vec![TokKind::DivEq]);
1859 assert_eq!(kinds(".⩴"), vec![TokKind::Dot, TokKind::UniAssign]);
1861 }
1862}