lex_lua/
lib.rs

1use std::{borrow::Cow, iter::Peekable};
2
3use bstr::{BStr, ByteSlice, Chars};
4
5pub struct Lexer<'a> {
6    buffer: Peekable<Chars<'a>>,
7    original: &'a BStr,
8    pos: usize,
9}
10
11pub struct SpannedLexer<'a> {
12    inner: Lexer<'a>,
13}
14
15impl<'a> SpannedLexer<'a> {
16    pub fn new(s: &'a [u8]) -> Self {
17        Self {
18            inner: Lexer::new(s),
19        }
20    }
21}
22
23impl<'a> Lexer<'a> {
24    pub fn new(s: &'a [u8]) -> Self {
25        let bs: &BStr = s.into();
26        Self {
27            buffer: bs.chars().peekable(),
28            original: bs,
29            pos: 0,
30        }
31    }
32
33    pub fn current_pos(&self) -> usize {
34        self.pos
35    }
36
37    fn next_char(&mut self) -> Option<char> {
38        let c = self.buffer.next()?;
39        if c as u32 == 0xFFFD {
40            self.pos += 1;
41        } else {
42            self.pos += c.len_utf8();
43        }
44        Some(c)
45    }
46
47    pub fn next_token(&mut self) -> Option<Token<'a>> {
48        let next = if self.pos == 0 && self.eat('#') {
49            return Some(self.comment(1));
50        } else {
51            self.skip_whitespace();
52            let next = self.next_char()?;
53            next
54        };
55        if next.is_digit(10) {
56            return Some(self.numeral(next));
57        }
58        if next == '"' || next == '\'' {
59            return Some(self.literal_string(next));
60        }
61
62        if next.is_ascii_alphabetic() || next == '_' {
63            return Some(self.name(next));
64        }
65
66        Some(self.punct(next))
67    }
68
69    fn next_spanned(&mut self) -> Option<Item<'a>> {
70        let (next, start) = if self.pos == 0 && self.eat('#') {
71            let token = self.comment(1);
72            return Some(Item::new(token, 0, self.pos));
73        } else {
74            self.skip_whitespace();
75            let start = self.pos;
76            let next = self.next_char()?;
77            (next, start)
78        };
79        if next.is_digit(10) {
80            let token = self.numeral(next);
81            return Some(Item::new(token, start, self.pos));
82        }
83        if next == '"' || next == '\'' {
84            let token = self.literal_string(next);
85            return Some(Item::new(token, start, self.pos));
86        }
87
88        if next.is_ascii_alphabetic() || next == '_' {
89            let token = self.name(next);
90            return Some(Item::new(token, start, self.pos));
91        }
92        let token = self.punct(next);
93        Some(Item::new(token, start, self.pos))
94    }
95
96    fn numeral(&mut self, c: char) -> Token<'a> {
97        let start = self.pos - 1;
98        if c == '0' && self.eat_ascii_ignore_case('x') {
99            while self.at_digit(16) {
100                let _ = self.next_char();
101            }
102            if self.eat('.') {
103                while self.at_digit(16) {
104                    let _ = self.next_char();
105                }
106            }
107            if self.eat_ascii_ignore_case('e') {
108                let _ = self.eat('-') || self.eat('+');
109                while self.at_digit(16) {
110                    let _ = self.next_char();
111                }
112            }
113            if self.eat_ascii_ignore_case('p') {
114                let _ = self.eat('-') || self.eat('+');
115                while self.at_digit(16) {
116                    let _ = self.next_char();
117                }
118            }
119        } else {
120            while self.at_digit(10) {
121                let _ = self.next_char();
122            }
123            if c != '.' && self.eat('.') {
124                while self.at_digit(10) {
125                    let _ = self.next_char();
126                }
127            }
128            if self.eat_ascii_ignore_case('e') {
129                let _ = self.eat('-') || self.eat('+');
130                while self.at_digit(10) {
131                    let _ = self.next_char();
132                }
133            }
134        }
135        Token::numeral(&self.original[start..self.pos])
136    }
137
138    fn literal_string(&mut self, c: char) -> Token<'a> {
139        let start = self.pos - 1;
140
141        if c == '=' || c == '[' {
142            let mut eq_ct = 0;
143            while self.eat('=') {
144                eq_ct += 1;
145            }
146            self.seek_long_string_end(eq_ct);
147        } else if c == '"' || c == '\'' {
148            let mut escaped = false;
149            while !self.at(c) || escaped {
150                if let Some(ch) = self.next_char() {
151                    escaped = ch == '\\' && !escaped;
152                } else {
153                    return Token::Unknown(Cow::Borrowed(&self.original[start..self.pos]));
154                }
155            }
156            self.eat(c);
157        }
158        Token::LiteralString(Cow::Borrowed(&self.original[start..self.pos]))
159    }
160
161    fn seek_long_string_end(&mut self, eq_ct: usize) {
162        'retry: loop {
163            while !self.eat(']') && !self.at_end() {
164                let _ = self.next_char();
165            }
166            let mut found_eq = 0;
167            while found_eq < eq_ct {
168                if self.eat('=') {
169                    found_eq += 1;
170                } else {
171                    continue 'retry;
172                }
173            }
174            if self.eat(']') {
175                return;
176            }
177        }
178    }
179
180    fn name(&mut self, c: char) -> Token<'a> {
181        let start = self.pos - 1;
182        match c {
183            'a' => {
184                if self.eat('n') {
185                    if self.eat('d') {
186                        if !self.at_name_cont() {
187                            return Token::Keyword(Keyword::And);
188                        }
189                    }
190                }
191            }
192            'b' => {
193                if self.eat('r') {
194                    if self.eat('e') {
195                        if self.eat('a') {
196                            if self.eat('k') {
197                                if !self.at_name_cont() {
198                                    return Token::Keyword(Keyword::Break);
199                                }
200                            }
201                        }
202                    }
203                }
204            }
205            'd' => {
206                if self.eat('o') {
207                    if !self.at_name_cont() {
208                        return Token::Keyword(Keyword::Do);
209                    }
210                }
211            }
212            'e' => {
213                if self.eat('l') {
214                    if self.eat('s') {
215                        if self.eat('e') {
216                            if !self.at_name_cont() {
217                                return Token::Keyword(Keyword::Else);
218                            } else if self.eat('i') {
219                                if self.eat('f') {
220                                    if !self.at_name_cont() {
221                                        return Token::Keyword(Keyword::ElseIf);
222                                    }
223                                }
224                            }
225                        }
226                    }
227                } else if self.eat('n') {
228                    if self.eat('d') {
229                        if !self.at_name_cont() {
230                            return Token::Keyword(Keyword::End);
231                        }
232                    }
233                }
234            }
235            'f' => {
236                if self.eat('a') {
237                    if self.eat('l') {
238                        if self.eat('s') {
239                            if self.eat('e') {
240                                if !self.at_name_cont() {
241                                    return Token::Keyword(Keyword::False);
242                                }
243                            }
244                        }
245                    }
246                } else if self.eat('o') {
247                    if self.eat('r') {
248                        if !self.at_name_cont() {
249                            return Token::Keyword(Keyword::For);
250                        }
251                    }
252                } else if self.eat('u') {
253                    if self.eat('n') {
254                        if self.eat('c') {
255                            if self.eat('t') {
256                                if self.eat('i') {
257                                    if self.eat('o') {
258                                        if self.eat('n') {
259                                            if !self.at_name_cont() {
260                                                return Token::Keyword(Keyword::Function);
261                                            }
262                                        }
263                                    }
264                                }
265                            }
266                        }
267                    }
268                }
269            }
270            'g' => {
271                if self.eat('o') {
272                    if self.eat('t') {
273                        if self.eat('o') {
274                            if !self.at_name_cont() {
275                                return Token::Keyword(Keyword::GoTo);
276                            }
277                        }
278                    }
279                }
280            }
281            'i' => {
282                if self.eat('n') {
283                    if !self.at_name_cont() {
284                        return Token::Keyword(Keyword::In);
285                    }
286                } else if self.eat('f') {
287                    if !self.at_name_cont() {
288                        return Token::Keyword(Keyword::If);
289                    }
290                }
291            }
292            'l' => {
293                if self.eat('o') {
294                    if self.eat('c') {
295                        if self.eat('a') {
296                            if self.eat('l') {
297                                if !self.at_name_cont() {
298                                    return Token::Keyword(Keyword::Local);
299                                }
300                            }
301                        }
302                    }
303                }
304            }
305            'n' => {
306                if self.eat('i') {
307                    if self.eat('l') {
308                        if !self.at_name_cont() {
309                            return Token::Keyword(Keyword::Nil);
310                        }
311                    }
312                } else if self.eat('o') {
313                    if self.eat('t') {
314                        if !self.at_name_cont() {
315                            return Token::Keyword(Keyword::Not);
316                        }
317                    }
318                }
319            }
320            'o' => {
321                if self.eat('r') {
322                    if !self.at_name_cont() {
323                        return Token::Keyword(Keyword::Or);
324                    }
325                }
326            }
327            'r' => {
328                if self.eat('e') {
329                    if self.eat('p') {
330                        if self.eat('e') {
331                            if self.eat('a') {
332                                if self.eat('t') {
333                                    if !self.at_name_cont() {
334                                        return Token::Keyword(Keyword::Repeat);
335                                    }
336                                }
337                            }
338                        }
339                    } else if self.eat('t') {
340                        if self.eat('u') {
341                            if self.eat('r') {
342                                if self.eat('n') {
343                                    if !self.at_name_cont() {
344                                        return Token::Keyword(Keyword::Return);
345                                    }
346                                }
347                            }
348                        }
349                    }
350                }
351            }
352            't' => {
353                if self.eat('h') {
354                    if self.eat('e') {
355                        if self.eat('n') {
356                            if !self.at_name_cont() {
357                                return Token::Keyword(Keyword::Then);
358                            }
359                        }
360                    }
361                } else if self.eat('r') {
362                    if self.eat('u') {
363                        if self.eat('e') {
364                            if !self.at_name_cont() {
365                                return Token::Keyword(Keyword::True);
366                            }
367                        }
368                    }
369                }
370            }
371            'u' => {
372                if self.eat('n') {
373                    if self.eat('t') {
374                        if self.eat('i') {
375                            if self.eat('l') {
376                                if !self.at_name_cont() {
377                                    return Token::Keyword(Keyword::Until);
378                                }
379                            }
380                        }
381                    }
382                }
383            }
384            'w' => {
385                if self.eat('h') {
386                    if self.eat('i') {
387                        if self.eat('l') {
388                            if self.eat('e') {
389                                if !self.at_name_cont() {
390                                    return Token::Keyword(Keyword::While);
391                                }
392                            }
393                        }
394                    }
395                }
396            }
397            _ => {
398                if !c.is_ascii_alphanumeric() && c != '_' {
399                    return Token::Unknown(Cow::Borrowed(&self.original[start..self.pos]));
400                }
401            }
402        }
403        while self.at_name_cont() {
404            let _ = self.next_char();
405        }
406        Token::name(&self.original[start..self.pos])
407    }
408
409    fn punct(&mut self, c: char) -> Token<'a> {
410        let p = match c {
411            '+' => Punct::Plus,
412            '-' => {
413                if self.eat('-') {
414                    return self.comment(2);
415                } else {
416                    Punct::Minus
417                }
418            }
419            '*' => Punct::Asterisk,
420            '%' => Punct::Percent,
421            '^' => Punct::Caret,
422            '#' => Punct::Hash,
423            '&' => Punct::Ampersand,
424            '~' => {
425                if self.eat('=') {
426                    Punct::TildeEqual
427                } else {
428                    Punct::Tilde
429                }
430            }
431            '|' => Punct::Pipe,
432            '(' => Punct::OpenParen,
433            ')' => Punct::CloseParen,
434            '{' => Punct::OpenBrace,
435            '}' => Punct::CloseBrace,
436            ';' => Punct::SemiColon,
437            ',' => Punct::Comma,
438            ']' => Punct::CloseBracket,
439            '.' => {
440                if self.at_digit(10) {
441                    return self.numeral('.');
442                } else if self.eat('.') {
443                    if self.eat('.') {
444                        Punct::Ellipsis
445                    } else {
446                        Punct::DoubleDot
447                    }
448                } else {
449                    Punct::Dot
450                }
451            }
452            ':' => {
453                if self.eat(':') {
454                    Punct::DoubleColon
455                } else {
456                    Punct::Colon
457                }
458            }
459            '/' => {
460                if self.eat('/') {
461                    Punct::DoubleForwardSlash
462                } else {
463                    Punct::ForwardSlash
464                }
465            }
466            '=' => {
467                if self.eat('=') {
468                    Punct::DoubleEqual
469                } else {
470                    Punct::Equal
471                }
472            }
473            '<' => {
474                if self.eat('<') {
475                    Punct::DoubleLessThan
476                } else if self.eat('=') {
477                    Punct::LessThanEqual
478                } else {
479                    Punct::LessThan
480                }
481            }
482            '>' => {
483                if self.eat('>') {
484                    Punct::DoubleGreaterThan
485                } else if self.eat('=') {
486                    Punct::GreaterThanEqual
487                } else {
488                    Punct::GreaterThan
489                }
490            }
491            '[' => {
492                if self.at('[') {
493                    return self.literal_string('[');
494                } else if self.at('=') {
495                    return self.literal_string('=');
496                } else {
497                    Punct::OpenBracket
498                }
499            }
500            _ => return Token::Unknown(Cow::Borrowed(&self.original[self.pos - 1..self.pos])),
501        };
502        Token::Punct(p)
503    }
504
505    fn comment(&mut self, start_len: usize) -> Token<'a> {
506        let start = self.pos - start_len;
507        if self.eat('[') {
508            let eq_ct = self.consume_long_comment_sep();
509            if self.eat('[') {
510                self.seek_long_string_end(eq_ct);
511                return Token::comment(&self.original[start..self.pos]);
512            }
513        }
514        while !self.at('\n') && !self.at_end() {
515            let _ = self.next_char();
516        }
517        Token::comment(&self.original[start..self.pos])
518    }
519
520    fn consume_long_comment_sep(&mut self) -> usize {
521        let mut ret = 0;
522        while self.eat('=') {
523            ret += 1;
524        }
525        ret
526    }
527
528    fn at(&mut self, c: char) -> bool {
529        if let Some(ch) = self.buffer.peek() {
530            *ch == c
531        } else {
532            false
533        }
534    }
535
536    fn eat(&mut self, c: char) -> bool {
537        if let Some(ch) = self.buffer.peek() {
538            if *ch == c {
539                let _ = self.next_char();
540                return true;
541            }
542        }
543        false
544    }
545
546    fn eat_ascii_ignore_case(&mut self, c: char) -> bool {
547        if let Some(ch) = self.buffer.peek() {
548            if ch.eq_ignore_ascii_case(&c) {
549                let _ = self.next_char();
550                return true;
551            }
552        }
553        false
554    }
555
556    fn skip_whitespace(&mut self) {
557        while let Some(ch) = self.buffer.peek() {
558            if ch.is_ascii_whitespace() {
559                let _ = self.next_char();
560            } else {
561                break;
562            }
563        }
564    }
565
566    fn at_name_cont(&mut self) -> bool {
567        if let Some(ch) = self.buffer.peek() {
568            ch.is_ascii_alphanumeric() || *ch == '_'
569        } else {
570            false
571        }
572    }
573
574    fn at_digit(&mut self, radix: u32) -> bool {
575        if let Some(ch) = self.buffer.peek() {
576            ch.is_digit(radix)
577        } else {
578            false
579        }
580    }
581
582    fn at_end(&mut self) -> bool {
583        self.buffer.peek().is_none()
584    }
585}
586
587impl<'a> std::iter::Iterator for Lexer<'a> {
588    type Item = Token<'a>;
589    fn next(&mut self) -> Option<Self::Item> {
590        self.next_token()
591    }
592}
593
594impl<'a> std::iter::Iterator for SpannedLexer<'a> {
595    type Item = Item<'a>;
596    fn next(&mut self) -> Option<Self::Item> {
597        self.inner.next_spanned()
598    }
599}
600
601#[derive(Debug, Clone, PartialEq, Eq)]
602pub struct Item<'a> {
603    pub token: Token<'a>,
604    pub span: Span,
605}
606
607impl<'a> AsRef<Span> for Item<'a> {
608    fn as_ref(&self) -> &Span {
609        &self.span
610    }
611}
612
613impl<'a> AsRef<Token<'a>> for Item<'a> {
614    fn as_ref(&self) -> &Token<'a> {
615        &self.token
616    }
617}
618
619impl<'a> PartialOrd for Item<'a> {
620    fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
621        self.span.partial_cmp(&other.span)
622    }
623}
624
625impl <'a> Ord for Item<'a> {
626    fn cmp(&self, other: &Self) -> std::cmp::Ordering {
627        self.partial_cmp(other).unwrap()
628    }
629}
630
631impl<'a> Item<'a> {
632    pub fn new(token: Token<'a>, start: usize, end: usize) -> Self {
633        Self {
634            token,
635            span: Span { start, end },
636        }
637    }
638}
639
640#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
641pub struct Span {
642    pub start: usize,
643    pub end: usize,
644}
645
646#[derive(Debug, Eq, PartialEq, Clone)]
647pub enum Token<'a> {
648    Name(Cow<'a, str>),
649    Numeral(Cow<'a, str>),
650    LiteralString(Cow<'a, BStr>),
651    Punct(Punct),
652    Keyword(Keyword),
653    Comment(Cow<'a, BStr>),
654    Unknown(Cow<'a, BStr>),
655}
656
657impl<'a> Token<'a> {
658    pub fn name(s: &'a BStr) -> Self {
659        Self::Name(s.to_str_lossy())
660    }
661    pub fn numeral(s: &'a BStr) -> Self {
662        Self::Numeral(s.to_str_lossy())
663    }
664    pub fn literal_string(s: &'a BStr) -> Self {
665        Self::LiteralString(Cow::Borrowed(s))
666    }
667    pub fn comment(s: &'a BStr) -> Self {
668        Self::Comment(Cow::Borrowed(s))
669    }
670}
671#[derive(Debug, Eq, PartialEq, Clone, Copy)]
672pub enum Punct {
673    Ampersand,
674    Asterisk,
675    Caret,
676    CloseBrace,
677    CloseBracket,
678    CloseParen,
679    Colon,
680    Comma,
681    Dot,
682    DoubleColon,
683    DoubleDot,
684    DoubleEqual,
685    DoubleForwardSlash,
686    DoubleGreaterThan,
687    DoubleLessThan,
688    Ellipsis,
689    Equal,
690    ForwardSlash,
691    GreaterThan,
692    GreaterThanEqual,
693    Hash,
694    LessThan,
695    LessThanEqual,
696    Minus,
697    OpenBrace,
698    OpenBracket,
699    OpenParen,
700    Percent,
701    Pipe,
702    Plus,
703    SemiColon,
704    Tilde,
705    TildeEqual,
706}
707
708impl std::str::FromStr for Punct {
709    type Err = String;
710
711    fn from_str(s: &str) -> Result<Self, Self::Err> {
712        let p = match s {
713            "&" => Punct::Ampersand,
714            "*" => Punct::Asterisk,
715            "^" => Punct::Caret,
716            "}" => Punct::CloseBrace,
717            "]" => Punct::CloseBracket,
718            ")" => Punct::CloseParen,
719            ":" => Punct::Colon,
720            "," => Punct::Comma,
721            "." => Punct::Dot,
722            "::" => Punct::DoubleColon,
723            ".." => Punct::DoubleDot,
724            "==" => Punct::DoubleEqual,
725            "//" => Punct::DoubleForwardSlash,
726            ">>" => Punct::DoubleGreaterThan,
727            "<<" => Punct::DoubleLessThan,
728            "..." => Punct::Ellipsis,
729            "=" => Punct::Equal,
730            "/" => Punct::ForwardSlash,
731            ">" => Punct::GreaterThan,
732            ">=" => Punct::GreaterThanEqual,
733            "#" => Punct::Hash,
734            "<" => Punct::LessThan,
735            "<=" => Punct::LessThanEqual,
736            "-" => Punct::Minus,
737            "{" => Punct::OpenBrace,
738            "[" => Punct::OpenBracket,
739            "(" => Punct::OpenParen,
740            "%" => Punct::Percent,
741            "|" => Punct::Pipe,
742            "+" => Punct::Plus,
743            ";" => Punct::SemiColon,
744            "~" => Punct::Tilde,
745            "~=" => Punct::TildeEqual,
746            _ => return Err(format!("Invalid punct: {:?}", s)),
747        };
748        Ok(p)
749    }
750}
751
752#[derive(Debug, Eq, PartialEq, Clone, Copy)]
753pub enum Keyword {
754    And,
755    Or,
756    Function,
757    Local,
758    For,
759    Do,
760    End,
761    Nil,
762    True,
763    False,
764    Not,
765    Return,
766    In,
767    While,
768    GoTo,
769    Break,
770    Repeat,
771    Then,
772    ElseIf,
773    Else,
774    Until,
775    If,
776}
777
778impl Keyword {
779    pub fn as_str(&self) -> &str {
780        match self {
781            Self::And => "and",
782            Self::Or => "or",
783            Self::Function => "function",
784            Self::Local => "local",
785            Self::For => "for",
786            Self::Do => "do",
787            Self::End => "end",
788            Self::Nil => "nil",
789            Self::True => "true",
790            Self::False => "false",
791            Self::Not => "not",
792            Self::Return => "return",
793            Self::In => "in",
794            Self::While => "while",
795            Self::GoTo => "goto",
796            Self::Break => "break",
797            Self::Repeat => "repeat",
798            Self::Then => "then",
799            Self::ElseIf => "elseif",
800            Self::Else => "else",
801            Self::Until => "until",
802            Self::If => "if",
803        }
804    }
805}
806
807#[cfg(test)]
808mod test {
809    use super::*;
810
811    #[test]
812    fn numerals() {
813        let strings = &[
814            "3",
815            "345",
816            "0xff",
817            "0xBEBADA",
818            "3.0",
819            "3.1416",
820            "314.16e-2",
821            "0.31416E1",
822            "34e1",
823            "0x0.1E",
824            "0xA23p-4",
825            "0X1.921FB54442D18P+1",
826            ".01",
827        ];
828        for &string in strings {
829            let mut t = Lexer::new(string.as_bytes());
830            assert_eq!(
831                t.next_token().unwrap(),
832                Token::Numeral(Cow::Borrowed(string.into()))
833            )
834        }
835    }
836
837    #[test]
838    fn puncts() {
839        let strings = &[
840            "+", "-", "*", "/", "//", "^", "%", "&", "~", "|", ">>", "<<", ".", "..", "...", "=",
841            "<", "<=", ">", ">=", "==", "~=", "#", "~", "(", ")", "[", "]", "{", "}", ",", ":",
842            "::", ";",
843        ];
844        for &string in strings {
845            println!("testing {:?}", string);
846            // Prefix here to avoid the len operator from
847            // being interpreted as a comment
848            let prefixed = format!("\n{}", string);
849            let mut t = Lexer::new(prefixed.as_bytes());
850            assert_eq!(
851                t.next_token().unwrap(),
852                Token::Punct(string.parse().unwrap())
853            )
854        }
855        let mut t = Lexer::new(b"$");
856        assert_eq!(
857            t.next_token().unwrap(),
858            Token::Unknown(Cow::Borrowed("$".into())),
859        )
860    }
861
862    #[test]
863    fn keywords() {
864        let keywords = &[
865            Keyword::And,
866            Keyword::Or,
867            Keyword::Function,
868            Keyword::Local,
869            Keyword::For,
870            Keyword::Do,
871            Keyword::End,
872            Keyword::Nil,
873            Keyword::True,
874            Keyword::False,
875            Keyword::Not,
876            Keyword::Return,
877            Keyword::In,
878            Keyword::While,
879            Keyword::GoTo,
880            Keyword::Break,
881            Keyword::Repeat,
882            Keyword::Then,
883            Keyword::ElseIf,
884            Keyword::Else,
885            Keyword::Until,
886            Keyword::If,
887        ];
888        for &keyword in keywords {
889            println!("testing {:?}", keyword.as_str());
890            let mut t = Lexer::new(keyword.as_str().as_bytes());
891            assert_eq!(
892                t.next_token().unwrap(),
893                Token::Keyword(keyword)
894            )
895        }
896        let mut t = Lexer::new(b"$");
897        assert_eq!(
898            t.next_token().unwrap(),
899            Token::Unknown(Cow::Borrowed("$".into())),
900        )
901    }
902
903    #[test]
904    fn literal_string() {
905        let strings = &[
906            r#"'alo\n123"'"#,
907            r#""alo\n123\"""#,
908            r#"[[alo
909123"]]"#,
910            r#"[==[
911alo
912123"]==]"#,
913            r#"[==[
914alo
915123"]=]==]"#,
916        ];
917        for (i, &s) in strings.iter().enumerate() {
918            println!("{} testing {:?}", i, s);
919            let mut t = Lexer::new(s.as_bytes());
920            assert_eq!(
921                t.next_token().unwrap(),
922                Token::LiteralString(Cow::Borrowed(s.into()))
923            );
924        }
925    }
926
927    #[test]
928    fn comments() {
929        let strings = &[
930            "#!hashbang",
931            "--single line comment",
932            "--[[multi
933            line comment]]",
934            "--[==[
935                [[multi with seps]]]]]]]
936            ]==]",
937        ];
938        for &s in strings {
939            let mut t = Lexer::new(s.as_bytes());
940            assert_eq!(
941                t.next_token().unwrap(),
942                Token::Comment(Cow::Borrowed(s.into()))
943            )
944        }
945    }
946    #[test]
947    fn strange_comments_interspersed() {
948        let lua = "print(--[[ hello ]]'hello world'--[[world]])";
949        let expected = &[
950            Token::Name(Cow::Borrowed("print")),
951            Token::Punct(Punct::OpenParen),
952            Token::Comment(Cow::Borrowed("--[[ hello ]]".into())),
953            Token::literal_string("'hello world'".into()),
954            Token::Comment(Cow::Borrowed("--[[world]]".into())),
955            Token::Punct(Punct::CloseParen),
956        ];
957        let mut t = Lexer::new(lua.as_bytes());
958        for tok in expected {
959            assert_eq!(
960                t.next_token().unwrap(),
961                *tok
962            )
963        }
964    }
965}
lex_lua/lib.rs

lex_lua/
lib.rs
