1#[cfg(all(feature = "no_std", not(feature = "std")))]
2use alloc::{format, string::String, vec::Vec};
3
4use crate::error::NyblError;
5
6pub(crate) const I64_MIN_MAGNITUDE: u64 = (i64::MAX as u64) + 1;
7pub(crate) const I64_MIN_MAGNITUDE_TEXT: &str = "9223372036854775808";
8
9pub(crate) fn integer_literal_out_of_range(digits: &str) -> String {
10 format!("Integer literal out of range for i64: {digits}")
11}
12
13#[derive(Debug, Clone, PartialEq)]
14pub enum StringPart {
15 Literal(String),
16 Variable(String),
17}
18
19#[derive(Debug, Clone, PartialEq)]
20pub enum Token {
21 Int(i64),
27 IntMinMagnitude,
32 Number(f64),
33 Str(String),
34 StringInterp(Vec<StringPart>),
35 True,
36 False,
37 None,
38
39 Ident(String),
41 Let,
42 Const,
43 Fn,
44 Pub,
45 Return,
46 If,
47 Else,
48 While,
49 For,
50 In,
51 Repeat,
52 Break,
53 Continue,
54 Use,
55 As,
56 Struct,
57 Enum,
58 Match,
59 Try,
60 Ref,
61
62 Plus,
64 Minus,
65 Star,
66 Slash,
67 Percent,
68 EqEq,
69 BangEq,
70 Lt,
71 Gt,
72 LtEq,
73 GtEq,
74 AmpAmp,
75 PipePipe,
76 Bang,
77 Eq,
78 PlusEq,
79 MinusEq,
80 StarEq,
81 SlashEq,
82 PercentEq,
83
84 LParen,
86 RParen,
87 LBracket,
88 RBracket,
89 LBrace,
90 RBrace,
91 Comma,
92 Colon,
93 ColonColon,
94 Dot,
95 DotDot,
96 Semicolon,
97 FatArrow,
98 Pipe,
99
100 Newline,
102
103 Eof,
104}
105
106macro_rules! define_keywords {
111 ($($spelling:literal => $variant:ident),+ $(,)?) => {
112 impl Token {
113 pub(crate) fn from_keyword(value: &str) -> Option<Self> {
114 match value {
115 $($spelling => Some(Self::$variant),)+
116 _ => None,
117 }
118 }
119
120 pub(crate) fn keyword_name(&self) -> Option<&'static str> {
121 match self {
122 $(Self::$variant => Some($spelling),)+
123 _ => None,
124 }
125 }
126 }
127
128 #[cfg(test)]
129 pub(crate) const KEYWORD_NAMES: &[&str] = &[$($spelling),+];
130 };
131}
132
133define_keywords! {
134 "let" => Let,
135 "const" => Const,
136 "fn" => Fn,
137 "pub" => Pub,
138 "return" => Return,
139 "if" => If,
140 "else" => Else,
141 "while" => While,
142 "for" => For,
143 "in" => In,
144 "repeat" => Repeat,
145 "break" => Break,
146 "continue" => Continue,
147 "use" => Use,
148 "as" => As,
149 "struct" => Struct,
150 "enum" => Enum,
151 "match" => Match,
152 "try" => Try,
153 "ref" => Ref,
154 "true" => True,
155 "false" => False,
156 "none" => None,
157}
158
159#[derive(Debug, Clone)]
160pub struct SpannedToken {
161 pub token: Token,
162 pub line: u32,
163 pub column: u32,
167}
168
169pub fn module_load_steps(tokens: &[SpannedToken]) -> u64 {
172 tokens
173 .iter()
174 .filter(|token| !matches!(token.token, Token::Eof))
175 .count() as u64
176}
177
178pub fn lex(source: &str) -> Result<Vec<SpannedToken>, NyblError> {
179 let mut lexer = Lexer::new(source);
180 let raw = lexer.lex_all()?;
181 Ok(insert_semicolons(raw))
182}
183
184fn triggers_semicolon(token: &Token) -> bool {
185 matches!(
186 token,
187 Token::Ident(_)
188 | Token::Int(_)
189 | Token::IntMinMagnitude
190 | Token::Number(_)
191 | Token::Str(_)
192 | Token::StringInterp(_)
193 | Token::True
194 | Token::False
195 | Token::None
196 | Token::Break
197 | Token::Continue
198 | Token::Return
199 | Token::RParen
200 | Token::RBracket
201 | Token::RBrace
202 )
203}
204
205fn supports_dot_continuation(token: &Token) -> bool {
206 matches!(
207 token,
208 Token::Ident(_)
209 | Token::Int(_)
210 | Token::IntMinMagnitude
211 | Token::Number(_)
212 | Token::Str(_)
213 | Token::StringInterp(_)
214 | Token::True
215 | Token::False
216 | Token::None
217 | Token::RParen
218 | Token::RBracket
219 | Token::RBrace
220 )
221}
222
223#[derive(Clone, Copy, PartialEq, Eq)]
224enum Delimiter {
225 Paren,
226 Bracket,
227 Brace,
228}
229
230fn insert_semicolons(raw: Vec<SpannedToken>) -> Vec<SpannedToken> {
231 let mut result: Vec<SpannedToken> = Vec::new();
232 let mut delimiters: Vec<Delimiter> = Vec::new();
233 let mut tokens = raw.into_iter().peekable();
234 while let Some(token) = tokens.next() {
235 if token.token == Token::Newline {
236 while tokens
239 .peek()
240 .is_some_and(|next| next.token == Token::Newline)
241 {
242 tokens.next();
243 }
244 if let Some(last) = result.last() {
245 let inside_soft_delimiter = matches!(
249 delimiters.last(),
250 Some(Delimiter::Paren | Delimiter::Bracket)
251 );
252 let next = tokens.peek().map(|next| &next.token);
253 let before_closing_delimiter =
254 matches!(next, Some(Token::RParen | Token::RBracket | Token::RBrace));
255 let before_else =
260 matches!(last.token, Token::RBrace) && matches!(next, Some(Token::Else));
261 let before_dot_continuation =
262 matches!(next, Some(Token::Dot)) && supports_dot_continuation(&last.token);
263 if !inside_soft_delimiter
264 && !before_closing_delimiter
265 && !before_else
266 && !before_dot_continuation
267 && triggers_semicolon(&last.token)
268 {
269 result.push(SpannedToken {
270 token: Token::Semicolon,
271 line: token.line,
272 column: token.column,
273 });
274 }
275 }
276 } else {
277 match &token.token {
278 Token::LParen => delimiters.push(Delimiter::Paren),
279 Token::LBracket => delimiters.push(Delimiter::Bracket),
280 Token::LBrace => delimiters.push(Delimiter::Brace),
281 Token::RParen if delimiters.last() == Some(&Delimiter::Paren) => {
282 delimiters.pop();
283 }
284 Token::RBracket if delimiters.last() == Some(&Delimiter::Bracket) => {
285 delimiters.pop();
286 }
287 Token::RBrace if delimiters.last() == Some(&Delimiter::Brace) => {
288 delimiters.pop();
289 }
290 _ => {}
291 }
292 result.push(token);
293 }
294 }
295 result
296}
297
298struct SourceCursor<'source> {
303 source: &'source str,
304 chars: core::str::CharIndices<'source>,
305 current: Option<(usize, char)>,
306 next: Option<(usize, char)>,
307 byte_pos: usize,
308}
309
310impl<'source> SourceCursor<'source> {
311 fn new(source: &'source str) -> Self {
312 let mut chars = source.char_indices();
313 let current = chars.next();
314 let next = chars.next();
315 Self {
316 source,
317 chars,
318 current,
319 next,
320 byte_pos: 0,
321 }
322 }
323
324 fn peek(&self) -> Option<char> {
325 self.current.map(|(_, ch)| ch)
326 }
327
328 fn peek_next(&self) -> Option<char> {
329 self.next.map(|(_, ch)| ch)
330 }
331
332 fn advance(&mut self) -> Option<char> {
333 let (_, ch) = self.current?;
334 self.current = self.next;
335 self.next = self.chars.next();
336 self.byte_pos = self
337 .current
338 .map(|(byte_pos, _)| byte_pos)
339 .unwrap_or(self.source.len());
340 Some(ch)
341 }
342}
343
344struct Lexer<'source> {
345 cursor: SourceCursor<'source>,
346 line: u32,
347 column: u32,
350}
351
352impl<'source> Lexer<'source> {
353 fn new(source: &'source str) -> Self {
354 Self {
355 cursor: SourceCursor::new(source),
356 line: 1,
357 column: 1,
358 }
359 }
360
361 fn peek(&self) -> Option<char> {
362 self.cursor.peek()
363 }
364
365 fn peek_next(&self) -> Option<char> {
366 self.cursor.peek_next()
367 }
368
369 fn advance(&mut self) -> Option<char> {
370 let ch = self.cursor.advance()?;
371 if ch == '\n' {
372 self.column = 1;
378 } else {
379 self.column += 1;
380 }
381 Some(ch)
382 }
383
384 fn error(&self, message: impl Into<String>) -> NyblError {
385 NyblError {
386 line: Some(self.line),
387 column: Some(self.column),
388 message: message.into(),
389 friendly_hint: None,
390 source_context: None,
391 is_fatal: false,
392 is_try_return: false,
393 }
394 }
395
396 fn error_at(&self, line: u32, column: u32, message: impl Into<String>) -> NyblError {
397 NyblError {
398 line: Some(line),
399 column: Some(column),
400 message: message.into(),
401 friendly_hint: None,
402 source_context: None,
403 is_fatal: false,
404 is_try_return: false,
405 }
406 }
407
408 fn error_with_hint(&self, message: impl Into<String>, hint: impl Into<String>) -> NyblError {
409 NyblError {
410 line: Some(self.line),
411 column: Some(self.column),
412 message: message.into(),
413 friendly_hint: Some(hint.into()),
414 source_context: None,
415 is_fatal: false,
416 is_try_return: false,
417 }
418 }
419
420 fn lex_all(&mut self) -> Result<Vec<SpannedToken>, NyblError> {
421 let mut tokens = Vec::new();
422
423 loop {
424 while let Some(ch) = self.peek() {
426 if ch == ' ' || ch == '\t' || ch == '\r' {
427 self.advance();
428 } else {
429 break;
430 }
431 }
432
433 let Some(ch) = self.peek() else {
434 tokens.push(SpannedToken {
435 token: Token::Eof,
436 line: self.line,
437 column: self.column,
438 });
439 break;
440 };
441
442 let line = self.line;
446 let column = self.column;
447
448 match ch {
449 '\n' => {
450 self.advance();
451 self.line += 1;
452 tokens.push(SpannedToken {
453 token: Token::Newline,
454 line,
455 column,
456 });
457 }
458
459 '"' => {
460 tokens.push(SpannedToken {
461 token: self.lex_string()?,
462 line,
463 column,
464 });
465 }
466
467 '0'..='9' => {
468 tokens.push(SpannedToken {
469 token: self.lex_number(line, column)?,
470 line,
471 column,
472 });
473 }
474
475 'a'..='z' | 'A'..='Z' | '_' => {
476 tokens.push(SpannedToken {
477 token: self.lex_ident_or_keyword(),
478 line,
479 column,
480 });
481 }
482
483 '+' => {
484 self.advance();
485 if self.peek() == Some('=') {
486 self.advance();
487 tokens.push(SpannedToken {
488 token: Token::PlusEq,
489 line,
490 column,
491 });
492 } else {
493 tokens.push(SpannedToken {
494 token: Token::Plus,
495 line,
496 column,
497 });
498 }
499 }
500 '-' => {
501 self.advance();
502 if self.peek() == Some('=') {
503 self.advance();
504 tokens.push(SpannedToken {
505 token: Token::MinusEq,
506 line,
507 column,
508 });
509 } else {
510 tokens.push(SpannedToken {
511 token: Token::Minus,
512 line,
513 column,
514 });
515 }
516 }
517 '*' => {
518 self.advance();
519 if self.peek() == Some('=') {
520 self.advance();
521 tokens.push(SpannedToken {
522 token: Token::StarEq,
523 line,
524 column,
525 });
526 } else {
527 tokens.push(SpannedToken {
528 token: Token::Star,
529 line,
530 column,
531 });
532 }
533 }
534 '/' => {
535 self.advance();
536 if self.peek() == Some('=') {
537 self.advance();
538 tokens.push(SpannedToken {
539 token: Token::SlashEq,
540 line,
541 column,
542 });
543 } else if self.peek() == Some('/') {
544 self.advance();
551 while let Some(c) = self.peek() {
552 if c == '\n' {
553 break;
554 }
555 self.advance();
556 }
557 } else {
558 tokens.push(SpannedToken {
559 token: Token::Slash,
560 line,
561 column,
562 });
563 }
564 }
565 '%' => {
566 self.advance();
567 if self.peek() == Some('=') {
568 self.advance();
569 tokens.push(SpannedToken {
570 token: Token::PercentEq,
571 line,
572 column,
573 });
574 } else {
575 tokens.push(SpannedToken {
576 token: Token::Percent,
577 line,
578 column,
579 });
580 }
581 }
582
583 '=' => {
584 self.advance();
585 if self.peek() == Some('=') {
586 self.advance();
587 tokens.push(SpannedToken {
588 token: Token::EqEq,
589 line,
590 column,
591 });
592 } else if self.peek() == Some('>') {
593 self.advance();
594 tokens.push(SpannedToken {
595 token: Token::FatArrow,
596 line,
597 column,
598 });
599 } else {
600 tokens.push(SpannedToken {
601 token: Token::Eq,
602 line,
603 column,
604 });
605 }
606 }
607 '!' => {
608 self.advance();
609 if self.peek() == Some('=') {
610 self.advance();
611 tokens.push(SpannedToken {
612 token: Token::BangEq,
613 line,
614 column,
615 });
616 } else {
617 tokens.push(SpannedToken {
618 token: Token::Bang,
619 line,
620 column,
621 });
622 }
623 }
624 '<' => {
625 self.advance();
626 if self.peek() == Some('=') {
627 self.advance();
628 tokens.push(SpannedToken {
629 token: Token::LtEq,
630 line,
631 column,
632 });
633 } else {
634 tokens.push(SpannedToken {
635 token: Token::Lt,
636 line,
637 column,
638 });
639 }
640 }
641 '>' => {
642 self.advance();
643 if self.peek() == Some('=') {
644 self.advance();
645 tokens.push(SpannedToken {
646 token: Token::GtEq,
647 line,
648 column,
649 });
650 } else {
651 tokens.push(SpannedToken {
652 token: Token::Gt,
653 line,
654 column,
655 });
656 }
657 }
658
659 '&' => {
660 self.advance();
661 if self.peek() == Some('&') {
662 self.advance();
663 tokens.push(SpannedToken {
664 token: Token::AmpAmp,
665 line,
666 column,
667 });
668 } else {
669 return Err(
670 self.error_with_hint("Unexpected `&`", "Did you mean `&&` (and)?")
671 );
672 }
673 }
674 '|' => {
675 self.advance();
676 if self.peek() == Some('|') {
677 self.advance();
678 tokens.push(SpannedToken {
679 token: Token::PipePipe,
680 line,
681 column,
682 });
683 } else {
684 tokens.push(SpannedToken {
689 token: Token::Pipe,
690 line,
691 column,
692 });
693 }
694 }
695
696 '(' => {
697 self.advance();
698 tokens.push(SpannedToken {
699 token: Token::LParen,
700 line,
701 column,
702 });
703 }
704 ')' => {
705 self.advance();
706 tokens.push(SpannedToken {
707 token: Token::RParen,
708 line,
709 column,
710 });
711 }
712 '[' => {
713 self.advance();
714 tokens.push(SpannedToken {
715 token: Token::LBracket,
716 line,
717 column,
718 });
719 }
720 ']' => {
721 self.advance();
722 tokens.push(SpannedToken {
723 token: Token::RBracket,
724 line,
725 column,
726 });
727 }
728 '{' => {
729 self.advance();
730 tokens.push(SpannedToken {
731 token: Token::LBrace,
732 line,
733 column,
734 });
735 }
736 '}' => {
737 self.advance();
738 tokens.push(SpannedToken {
739 token: Token::RBrace,
740 line,
741 column,
742 });
743 }
744 ',' => {
745 self.advance();
746 tokens.push(SpannedToken {
747 token: Token::Comma,
748 line,
749 column,
750 });
751 }
752 ':' => {
753 self.advance();
754 if self.peek() == Some(':') {
755 self.advance();
756 tokens.push(SpannedToken {
757 token: Token::ColonColon,
758 line,
759 column,
760 });
761 } else {
762 tokens.push(SpannedToken {
763 token: Token::Colon,
764 line,
765 column,
766 });
767 }
768 }
769 '.' => {
770 self.advance();
771 if self.peek() == Some('.') {
772 self.advance();
773 tokens.push(SpannedToken {
774 token: Token::DotDot,
775 line,
776 column,
777 });
778 } else {
779 tokens.push(SpannedToken {
780 token: Token::Dot,
781 line,
782 column,
783 });
784 }
785 }
786 ';' => {
787 self.advance();
788 tokens.push(SpannedToken {
789 token: Token::Semicolon,
790 line,
791 column,
792 });
793 }
794
795 _ => {
796 return Err(self.error(format!("I don't understand the character `{ch}`")));
797 }
798 }
799 }
800
801 Ok(tokens)
802 }
803
804 fn lex_number(&mut self, line: u32, column: u32) -> Result<Token, NyblError> {
805 let mut s = String::new();
806 while let Some(ch) = self.peek() {
807 if ch.is_ascii_digit() {
808 s.push(ch);
809 self.advance();
810 } else {
811 break;
812 }
813 }
814 let is_float =
819 if self.peek() == Some('.') && self.peek_next().is_some_and(|c| c.is_ascii_digit()) {
820 s.push('.');
821 self.advance();
822 while let Some(ch) = self.peek() {
823 if ch.is_ascii_digit() {
824 s.push(ch);
825 self.advance();
826 } else {
827 break;
828 }
829 }
830 true
831 } else {
832 false
833 };
834 if is_float {
835 let n: f64 = s
836 .parse()
837 .map_err(|_| self.error_at(line, column, format!("Invalid number: {s}")))?;
838 Ok(Token::Number(n))
839 } else {
840 match s.parse::<u64>() {
845 Ok(n) if n <= i64::MAX as u64 => Ok(Token::Int(n as i64)),
846 Ok(I64_MIN_MAGNITUDE) => Ok(Token::IntMinMagnitude),
847 _ => Err(self.error_at(line, column, integer_literal_out_of_range(&s))),
848 }
849 }
850 }
851
852 fn lex_ident_or_keyword(&mut self) -> Token {
853 let mut s = String::new();
854 while let Some(ch) = self.peek() {
855 if ch.is_ascii_alphanumeric() || ch == '_' {
856 s.push(ch);
857 self.advance();
858 } else {
859 break;
860 }
861 }
862 Token::from_keyword(&s).unwrap_or(Token::Ident(s))
863 }
864
865 fn lex_string(&mut self) -> Result<Token, NyblError> {
866 self.advance(); let mut parts: Vec<StringPart> = Vec::new();
868 let mut current = String::new();
869
870 loop {
871 match self.peek() {
872 None | Some('\n') => {
873 return Err(self.error_with_hint(
874 "This string is missing its closing `\"`",
875 "Every string needs to start and end with quotes.",
876 ));
877 }
878 Some('"') => {
879 self.advance();
880 break;
881 }
882 Some('\\') => {
883 self.advance();
884 match self.peek() {
885 Some('"') => {
886 current.push('"');
887 self.advance();
888 }
889 Some('\\') => {
890 current.push('\\');
891 self.advance();
892 }
893 Some('n') => {
894 current.push('\n');
895 self.advance();
896 }
897 Some('t') => {
898 current.push('\t');
899 self.advance();
900 }
901 Some('r') => {
902 current.push('\r');
907 self.advance();
908 }
909 Some('{') => {
910 current.push('{');
911 self.advance();
912 }
913 Some('}') => {
914 current.push('}');
915 self.advance();
916 }
917 Some(c) => {
918 return Err(self.error(format!("Unknown escape sequence `\\{c}`")));
919 }
920 None => {
921 return Err(self.error("Unexpected end of string after `\\`"));
922 }
923 }
924 }
925 Some('{')
926 if self
927 .peek_next()
928 .is_some_and(|c| c.is_ascii_alphabetic() || c == '_') =>
929 {
930 self.advance(); let mut var = String::new();
933 while let Some(ch) = self.peek() {
934 if ch.is_ascii_alphanumeric() || ch == '_' {
935 var.push(ch);
936 self.advance();
937 } else {
938 break;
939 }
940 }
941 if self.peek() != Some('}') {
942 return Err(self.error_with_hint(
943 format!("Missing `}}` after `{{{var}`"),
944 "String interpolation needs a closing `}`, like: \"{name}\"",
945 ));
946 }
947 self.advance(); if !current.is_empty() {
949 parts.push(StringPart::Literal(core::mem::take(&mut current)));
950 }
951 parts.push(StringPart::Variable(var));
952 }
953 Some(ch) => {
954 current.push(ch);
955 self.advance();
956 }
957 }
958 }
959
960 if parts.is_empty() {
961 Ok(Token::Str(current))
963 } else {
964 if !current.is_empty() {
965 parts.push(StringPart::Literal(current));
966 }
967 Ok(Token::StringInterp(parts))
968 }
969 }
970}
971
972#[cfg(test)]
973mod tests {
974 use super::*;
975
976 fn toks(code: &str) -> Vec<Token> {
978 lex(code)
979 .unwrap()
980 .into_iter()
981 .map(|t| t.token)
982 .filter(|t| !matches!(t, Token::Eof))
983 .collect()
984 }
985
986 fn lex_err(code: &str) -> String {
987 lex(code).unwrap_err().message
988 }
989
990 #[test]
991 fn source_cursor_borrows_utf8_and_advances_on_character_boundaries() {
992 let source = "é🙂x";
993 let mut lexer = Lexer::new(source);
994
995 assert_eq!(lexer.cursor.source.as_ptr(), source.as_ptr());
996 assert_eq!(lexer.cursor.byte_pos, 0);
997
998 assert_eq!(lexer.advance(), Some('é'));
999 assert_eq!(lexer.cursor.byte_pos, 'é'.len_utf8());
1000 assert_eq!(lexer.column, 2);
1001
1002 assert_eq!(lexer.advance(), Some('🙂'));
1003 assert_eq!(lexer.cursor.byte_pos, 'é'.len_utf8() + '🙂'.len_utf8());
1004 assert_eq!(lexer.column, 3);
1005 assert_eq!(lexer.peek(), Some('x'));
1006 }
1007
1008 #[test]
1009 fn utf8_string_contents_preserve_character_based_token_columns() {
1010 let tokens = lex("\"é🙂\" next").expect("source should lex");
1011 assert!(matches!(&tokens[0].token, Token::Str(value) if value == "é🙂"));
1012 assert_eq!((tokens[0].line, tokens[0].column), (1, 1));
1013 assert!(matches!(&tokens[1].token, Token::Ident(name) if name == "next"));
1014 assert_eq!((tokens[1].line, tokens[1].column), (1, 6));
1015 }
1016
1017 #[test]
1020 fn integer() {
1021 assert_eq!(toks("42"), vec![Token::Int(42)]);
1023 }
1024
1025 #[test]
1026 fn i64_min_magnitude_is_preserved_for_unary_minus_parsing() {
1027 assert_eq!(toks(I64_MIN_MAGNITUDE_TEXT), vec![Token::IntMinMagnitude]);
1028 }
1029
1030 #[test]
1031 fn integer_beyond_i64_min_magnitude_errors_at_the_literal_start() {
1032 let error = lex("\n 9223372036854775809").expect_err("magnitude must be rejected");
1033 assert_eq!(error.line, Some(2));
1034 assert_eq!(error.column, Some(3));
1035 assert_eq!(
1036 error.message,
1037 "Integer literal out of range for i64: 9223372036854775809"
1038 );
1039 }
1040
1041 #[test]
1042 fn float() {
1043 assert_eq!(toks("3.125"), vec![Token::Number(3.125)]);
1044 }
1045
1046 #[test]
1047 fn leading_zero_float() {
1048 assert_eq!(toks("0.5"), vec![Token::Number(0.5)]);
1049 }
1050
1051 #[test]
1054 fn plain_string() {
1055 assert_eq!(toks(r#""hello""#), vec![Token::Str("hello".into())]);
1056 }
1057
1058 #[test]
1059 fn escape_sequences() {
1060 assert_eq!(
1061 toks(r#""a\nb\t\\\"c""#),
1062 vec![Token::Str("a\nb\t\\\"c".into())]
1063 );
1064 }
1065
1066 #[test]
1067 fn escape_sequence_cr() {
1068 assert_eq!(toks(r#""a\rb""#), vec![Token::Str("a\rb".into())]);
1072 }
1073
1074 #[test]
1075 fn string_interpolation() {
1076 assert_eq!(
1077 toks(r#""hi {name}!""#),
1078 vec![Token::StringInterp(vec![
1079 StringPart::Literal("hi ".into()),
1080 StringPart::Variable("name".into()),
1081 StringPart::Literal("!".into()),
1082 ])]
1083 );
1084 }
1085
1086 #[test]
1087 fn string_interpolation_multiple_vars() {
1088 assert_eq!(
1089 toks(r#""{x},{y}""#),
1090 vec![Token::StringInterp(vec![
1091 StringPart::Variable("x".into()),
1092 StringPart::Literal(",".into()),
1093 StringPart::Variable("y".into()),
1094 ])]
1095 );
1096 }
1097
1098 #[test]
1099 fn unterminated_string() {
1100 assert!(lex_err(r#""hello"#).contains("missing its closing"));
1101 }
1102
1103 #[test]
1104 fn unknown_escape() {
1105 assert!(lex_err(r#""hello\q""#).contains("Unknown escape"));
1106 }
1107
1108 #[test]
1111 fn keywords() {
1112 assert_eq!(
1113 toks("let fn return if else while for in repeat break continue true false none"),
1114 vec![
1115 Token::Let,
1116 Token::Fn,
1117 Token::Return,
1118 Token::If,
1119 Token::Else,
1120 Token::While,
1121 Token::For,
1122 Token::In,
1123 Token::Repeat,
1124 Token::Break,
1125 Token::Continue,
1126 Token::True,
1127 Token::False,
1128 Token::None,
1129 ]
1130 );
1131 }
1132
1133 #[test]
1134 fn keyword_metadata_round_trips_the_lexer_vocabulary() {
1135 assert!(KEYWORD_NAMES.contains(&"const"));
1136
1137 for &name in KEYWORD_NAMES {
1138 let token = Token::from_keyword(name).expect("declared keyword should lex");
1139 assert_eq!(token.keyword_name(), Some(name));
1140 assert_eq!(toks(name), vec![token]);
1141 }
1142 }
1143
1144 #[test]
1145 fn identifiers() {
1146 assert_eq!(
1147 toks("foo bar_baz _x abc123"),
1148 vec![
1149 Token::Ident("foo".into()),
1150 Token::Ident("bar_baz".into()),
1151 Token::Ident("_x".into()),
1152 Token::Ident("abc123".into()),
1153 ]
1154 );
1155 }
1156
1157 #[test]
1160 fn single_char_ops() {
1161 assert_eq!(
1162 toks("+ - * / % = ! < > ( ) [ ] { } , : . ;"),
1163 vec![
1164 Token::Plus,
1165 Token::Minus,
1166 Token::Star,
1167 Token::Slash,
1168 Token::Percent,
1169 Token::Eq,
1170 Token::Bang,
1171 Token::Lt,
1172 Token::Gt,
1173 Token::LParen,
1174 Token::RParen,
1175 Token::LBracket,
1176 Token::RBracket,
1177 Token::LBrace,
1178 Token::RBrace,
1179 Token::Comma,
1180 Token::Colon,
1181 Token::Dot,
1182 Token::Semicolon,
1183 ]
1184 );
1185 }
1186
1187 #[test]
1188 fn double_char_ops() {
1189 assert_eq!(
1190 toks("== != <= >= && || += -= *= /= %="),
1191 vec![
1192 Token::EqEq,
1193 Token::BangEq,
1194 Token::LtEq,
1195 Token::GtEq,
1196 Token::AmpAmp,
1197 Token::PipePipe,
1198 Token::PlusEq,
1199 Token::MinusEq,
1200 Token::StarEq,
1201 Token::SlashEq,
1202 Token::PercentEq,
1203 ]
1204 );
1205 }
1206
1207 #[test]
1208 fn lone_ampersand_error() {
1209 assert!(lex_err("&x").contains("Unexpected `&`"));
1210 }
1211
1212 #[test]
1213 fn lone_pipe_lexes_as_or_pattern_separator() {
1214 assert_eq!(toks("|"), vec![Token::Pipe]);
1218 }
1219
1220 #[test]
1223 fn line_comment_skipped() {
1224 assert_eq!(
1225 toks("1 // comment\n2"),
1226 vec![Token::Int(1), Token::Semicolon, Token::Int(2)]
1227 );
1228 }
1229
1230 #[test]
1231 fn comment_at_end() {
1232 assert_eq!(toks("x // done"), vec![Token::Ident("x".into())]);
1233 }
1234
1235 #[test]
1236 fn hash_is_not_a_comment() {
1237 assert!(lex_err("x # nope").contains("don't understand"));
1240 }
1241
1242 #[test]
1245 fn auto_semi_after_ident() {
1246 assert_eq!(
1247 toks("x\ny"),
1248 vec![
1249 Token::Ident("x".into()),
1250 Token::Semicolon,
1251 Token::Ident("y".into()),
1252 ]
1253 );
1254 }
1255
1256 #[test]
1257 fn auto_semi_after_number() {
1258 assert_eq!(
1259 toks("42\n10"),
1260 vec![Token::Int(42), Token::Semicolon, Token::Int(10)]
1261 );
1262 }
1263
1264 #[test]
1265 fn auto_semi_after_rparen() {
1266 assert_eq!(
1267 toks("f()\ng()"),
1268 vec![
1269 Token::Ident("f".into()),
1270 Token::LParen,
1271 Token::RParen,
1272 Token::Semicolon,
1273 Token::Ident("g".into()),
1274 Token::LParen,
1275 Token::RParen,
1276 ]
1277 );
1278 }
1279
1280 #[test]
1281 fn auto_semi_after_rbrace() {
1282 assert_eq!(
1283 toks("{\n}\nx"),
1284 vec![
1285 Token::LBrace,
1286 Token::RBrace,
1287 Token::Semicolon,
1288 Token::Ident("x".into()),
1289 ]
1290 );
1291 }
1292
1293 #[test]
1294 fn no_semi_after_open_delim() {
1295 assert_eq!(toks("{\nx"), vec![Token::LBrace, Token::Ident("x".into()),]);
1296 }
1297
1298 #[test]
1299 fn no_semi_after_operator() {
1300 assert_eq!(
1301 toks("x +\ny"),
1302 vec![
1303 Token::Ident("x".into()),
1304 Token::Plus,
1305 Token::Ident("y".into()),
1306 ]
1307 );
1308 }
1309
1310 #[test]
1311 fn auto_semi_after_break_continue_return() {
1312 assert_eq!(
1313 toks("break\ncontinue\nreturn"),
1314 vec![
1315 Token::Break,
1316 Token::Semicolon,
1317 Token::Continue,
1318 Token::Semicolon,
1319 Token::Return,
1320 ]
1321 );
1322 }
1323
1324 #[test]
1325 fn auto_semi_after_true_false_none() {
1326 assert_eq!(
1327 toks("true\nfalse\nnone"),
1328 vec![
1329 Token::True,
1330 Token::Semicolon,
1331 Token::False,
1332 Token::Semicolon,
1333 Token::None,
1334 ]
1335 );
1336 }
1337
1338 #[test]
1339 fn no_auto_semi_inside_parentheses_or_brackets() {
1340 assert_eq!(
1341 toks("[\n1,\n2\n]\nf(\n3,\n4\n)"),
1342 vec![
1343 Token::LBracket,
1344 Token::Int(1),
1345 Token::Comma,
1346 Token::Int(2),
1347 Token::RBracket,
1348 Token::Semicolon,
1349 Token::Ident("f".into()),
1350 Token::LParen,
1351 Token::Int(3),
1352 Token::Comma,
1353 Token::Int(4),
1354 Token::RParen,
1355 ]
1356 );
1357 }
1358
1359 #[test]
1360 fn braces_inside_parentheses_keep_statement_separation() {
1361 assert_eq!(
1362 toks("(fn() {\nx\ny\n})"),
1363 vec![
1364 Token::LParen,
1365 Token::Fn,
1366 Token::LParen,
1367 Token::RParen,
1368 Token::LBrace,
1369 Token::Ident("x".into()),
1370 Token::Semicolon,
1371 Token::Ident("y".into()),
1372 Token::RBrace,
1373 Token::RParen,
1374 ]
1375 );
1376 }
1377
1378 #[test]
1379 fn braces_inside_brackets_keep_statement_separation() {
1380 assert_eq!(
1381 toks("[fn() {\nx\ny\n}]"),
1382 vec![
1383 Token::LBracket,
1384 Token::Fn,
1385 Token::LParen,
1386 Token::RParen,
1387 Token::LBrace,
1388 Token::Ident("x".into()),
1389 Token::Semicolon,
1390 Token::Ident("y".into()),
1391 Token::RBrace,
1392 Token::RBracket,
1393 ]
1394 );
1395 }
1396
1397 #[test]
1398 fn newline_before_closing_delimiter_does_not_insert_semicolon() {
1399 assert_eq!(
1400 toks("{\nx\ny\n}"),
1401 vec![
1402 Token::LBrace,
1403 Token::Ident("x".into()),
1404 Token::Semicolon,
1405 Token::Ident("y".into()),
1406 Token::RBrace,
1407 ]
1408 );
1409 }
1410
1411 #[test]
1412 fn newline_before_else_does_not_insert_semicolon() {
1413 assert_eq!(
1414 toks("if true { 1 }\nelse { 2 }"),
1415 vec![
1416 Token::If,
1417 Token::True,
1418 Token::LBrace,
1419 Token::Int(1),
1420 Token::RBrace,
1421 Token::Else,
1422 Token::LBrace,
1423 Token::Int(2),
1424 Token::RBrace,
1425 ]
1426 );
1427
1428 assert_eq!(
1429 toks("if true { 1 };\nelse { 2 }"),
1430 vec![
1431 Token::If,
1432 Token::True,
1433 Token::LBrace,
1434 Token::Int(1),
1435 Token::RBrace,
1436 Token::Semicolon,
1437 Token::Else,
1438 Token::LBrace,
1439 Token::Int(2),
1440 Token::RBrace,
1441 ]
1442 );
1443 }
1444
1445 #[test]
1446 fn leading_dot_continues_a_value_across_comments_and_blank_lines() {
1447 assert_eq!(
1448 toks("value\n// explain the chain\n\n.len()"),
1449 vec![
1450 Token::Ident("value".into()),
1451 Token::Dot,
1452 Token::Ident("len".into()),
1453 Token::LParen,
1454 Token::RParen,
1455 ]
1456 );
1457 }
1458
1459 #[test]
1460 fn return_newline_remains_a_bare_return() {
1461 assert_eq!(
1462 toks("return\nvalue"),
1463 vec![
1464 Token::Return,
1465 Token::Semicolon,
1466 Token::Ident("value".into()),
1467 ]
1468 );
1469 }
1470
1471 #[test]
1472 fn delimiter_characters_inside_strings_do_not_affect_asi() {
1473 assert_eq!(
1474 toks("\"([\"\nx\n\"{name}]\"\ny"),
1475 vec![
1476 Token::Str("([".into()),
1477 Token::Semicolon,
1478 Token::Ident("x".into()),
1479 Token::Semicolon,
1480 Token::StringInterp(vec![
1481 StringPart::Variable("name".into()),
1482 StringPart::Literal("]".into()),
1483 ]),
1484 Token::Semicolon,
1485 Token::Ident("y".into()),
1486 ]
1487 );
1488 }
1489
1490 #[test]
1491 fn unmatched_closing_delimiter_does_not_corrupt_depth() {
1492 assert_eq!(
1493 toks("]\nx"),
1494 vec![Token::RBracket, Token::Semicolon, Token::Ident("x".into()),]
1495 );
1496 }
1497
1498 #[test]
1501 fn line_numbers() {
1502 let tokens = lex("x\ny\nz").unwrap();
1503 let lines: Vec<u32> = tokens.iter().map(|t| t.line).collect();
1504 assert_eq!(lines, vec![1, 1, 2, 2, 3, 3]);
1506 }
1507
1508 #[test]
1511 fn unknown_char() {
1512 assert!(lex_err("@").contains("don't understand"));
1513 }
1514}