rustre_parser/
lexer.rs

1use crate::location::{Location, Span, Spanned};
2
3/// A spanned token
4pub type Tok<'a, 'f> = Spanned<'f, TokInfo<'a>>;
5
6#[derive(Clone, Debug, PartialEq)]
7/// A token
8pub enum TokInfo<'a> {
9    EOF,
10    Extern,
11    Unsafe,
12    And,
13    Arrow,
14    Assert,
15    Bar,
16    Bool,
17    CDots,
18    CloseBrace,
19    CloseBracket,
20    ClosePar,
21    CloseStaticPar,
22    Colon,
23    Coma,
24    Const,
25    Current,
26    Sharp,
27    Div,
28    DoubleColon,
29    Dot,
30    Equal,
31    Else,
32    Enum,
33    False,
34    Function,
35    Gt,
36    Gte,
37    Hat,
38    IConst(i64),
39    Ident(&'a str),
40    If,
41    Impl,
42    Int,
43    Let,
44    Lt,
45    Lte,
46    Merge,
47    Minus,
48    Mod,
49    Neq,
50    Node,
51    Nor,
52    Not,
53    OpenBrace,
54    OpenBracket,
55    OpenPar,
56    OpenStaticPar,
57    Operator,
58    Or,
59    Percent,
60    Plus,
61    Power,
62    Pre,
63    FBy,
64    RConst(f64),
65    Real,
66    Returns,
67    Semicolon,
68    Slash,
69    Star,
70    Step,
71    Struct,
72    Tel,
73    Then,
74    True,
75    Type,
76    Var,
77    When,
78    With,
79    Xor,
80    Model,
81    Package,
82    Needs,
83    Provides,
84    Uses,
85    Is,
86    Body,
87    End,
88    Include,
89    Str(&'a str),
90}
91
92#[derive(Copy, Clone, Eq, PartialEq)]
93enum Grammar {
94    Main,
95    Str,
96    InlineComment,
97    Comment(char),
98}
99
100#[cfg_attr(test, derive(PartialEq))]
101#[derive(Debug)]
102/// Lexing error
103pub enum Error {
104    UnclosedStr,
105    UnclosedComment,
106}
107
108/// A lexer for Lustre v6 files
109pub struct Lexer<'a, 'f> {
110    file: &'f str,
111    src: &'a str,
112}
113
114#[derive(PartialEq, Debug)]
115enum MatchResult<'a> {
116    NotYetMatched,
117    NoMatches,
118    Ambiguous(Vec<TokInfo<'a>>),
119    Match(LexMatch<'a>),
120    ManyMatches(Vec<TokInfo<'a>>),
121}
122
123#[derive(PartialEq, Debug)]
124enum LexMatch<'a> {
125    Tok(TokInfo<'a>),
126    Comment(&'a str),
127}
128
129impl<'a, 'f> Lexer<'a, 'f> {
130    /// Initializes a new lexer
131    ///
132    /// # Parameters
133    ///
134    /// - `file`: the path of the file
135    /// - `src`: the contents of the file
136    pub fn new(file: &'f str, src: &'a str) -> Self {
137        Lexer { file, src }
138    }
139
140    /// Generates a list of tokens from the file
141    pub fn lex(&mut self) -> Result<Vec<Tok>, Error> {
142        let total_len = self.src.len();
143        let mut res = Vec::with_capacity(total_len / 4);
144        let mut start = 0;
145        let mut end = 0;
146        let mut grammar = Grammar::Main;
147        let mut line = 1;
148        let mut col = 0;
149        while end < total_len {
150            let mut last_span = Span::default();
151            let mut last_match = MatchResult::NotYetMatched;
152            let mut curr_match = MatchResult::NotYetMatched;
153            let mut last_end = end;
154            while end <= total_len && curr_match != MatchResult::NoMatches {
155                let src_slice = &self.src[start..end];
156                match src_slice {
157                    "" => {}
158                    "--" => grammar = Grammar::InlineComment,
159                    "/*" => grammar = Grammar::Comment('/'),
160                    "(*" => grammar = Grammar::Comment(')'),
161                    "\"" => grammar = Grammar::Str,
162                    _ => {
163                        last_span = self.span(line, col, start, last_end - start);
164                        last_match = curr_match;
165                        curr_match = match grammar {
166                            Grammar::InlineComment => self.match_inline_comm(src_slice),
167                            Grammar::Comment(delim) => self.match_comm(src_slice, delim),
168                            Grammar::Str => self.match_str(src_slice),
169                            Grammar::Main => self.match_tokens(src_slice),
170                        };
171                        if curr_match != MatchResult::NotYetMatched {
172                            grammar = Grammar::Main;
173                        }
174                    }
175                }
176                if curr_match != MatchResult::NoMatches {
177                    last_end = end;
178                    end = self.next_char(end, 1);
179                }
180            }
181
182            let added_lines = self.src[last_span.start.pos as usize..last_span.end.pos as usize]
183                .chars()
184                .filter(|c| *c == '\n')
185                .count();
186            if added_lines != 0 {
187                line += added_lines;
188                col = 0;
189            }
190            let added_cols = self.src[last_span.start.pos as usize..last_span.end.pos as usize]
191                .lines()
192                .last()
193                .unwrap_or_default()
194                .len();
195            col += added_cols;
196
197            if end == total_len + 1 {
198                last_match = curr_match;
199            }
200
201            match last_match {
202                MatchResult::Match(LexMatch::Tok(m)) => {
203                    res.push(Spanned {
204                        span: last_span.clone(),
205                        item: m,
206                    });
207                    end = last_end;
208                }
209                MatchResult::ManyMatches(matches) | MatchResult::Ambiguous(matches) => {
210                    for m in matches {
211                        res.push(Spanned {
212                            span: last_span.clone(),
213                            item: m,
214                        });
215                    }
216                    end = last_end;
217                }
218                _ => {}
219            }
220            start = last_end;
221        }
222        if grammar == Grammar::Str {
223            Err(Error::UnclosedStr)
224        } else if let Grammar::Comment(_) = grammar {
225            Err(Error::UnclosedComment)
226        } else {
227            res.push(Spanned {
228                span: self.span(line, col, start, 0),
229                item: TokInfo::EOF,
230            });
231            Ok(res)
232        }
233    }
234
235    fn next_char(&self, from: usize, add: isize) -> usize {
236        Self::next_char_in(self.src, from, add)
237    }
238
239    fn next_char_in(src: &str, from: usize, add: isize) -> usize {
240        let mut new_pos = (from as isize) + add;
241        while !src.is_char_boundary(new_pos as usize) && (new_pos as usize) < src.len() {
242            new_pos += add.signum();
243        }
244        new_pos as usize
245    }
246
247    fn match_inline_comm(&self, src_slice: &'a str) -> MatchResult<'a> {
248        let len = src_slice.len();
249        let start = Self::next_char_in(src_slice, len, -1);
250        if &src_slice[start..] == "\n" {
251            MatchResult::Match(LexMatch::Comment(&src_slice[2..]))
252        } else {
253            MatchResult::NotYetMatched
254        }
255    }
256
257    fn match_comm(&self, src_slice: &'a str, delim: char) -> MatchResult<'a> {
258        let end = format!("*{}", delim);
259        let len = src_slice.len();
260        let start = Self::next_char_in(src_slice, len, -2);
261        if &src_slice[start..] == &end {
262            MatchResult::Match(LexMatch::Comment(&src_slice[2..start]))
263        } else {
264            MatchResult::NotYetMatched
265        }
266    }
267
268    fn match_str(&self, src_slice: &'a str) -> MatchResult<'a> {
269        let len = src_slice.len();
270        let start = Self::next_char_in(src_slice, len, -1);
271        if &src_slice[start..] == "\"" {
272            MatchResult::Match(LexMatch::Tok(TokInfo::Str(&src_slice[1..start])))
273        } else {
274            MatchResult::NotYetMatched
275        }
276    }
277
278    fn match_tokens(&self, src_slice: &'a str) -> MatchResult<'a> {
279        match src_slice {
280            " " | "\t" | "\n" | "\r" => MatchResult::NoMatches,
281            "," => MatchResult::Match(LexMatch::Tok(TokInfo::Coma)),
282            ";" => MatchResult::Match(LexMatch::Tok(TokInfo::Semicolon)),
283            "::" => MatchResult::Match(LexMatch::Tok(TokInfo::DoubleColon)),
284            ":" => MatchResult::Match(LexMatch::Tok(TokInfo::Colon)),
285            "->" => MatchResult::Match(LexMatch::Tok(TokInfo::Arrow)),
286            "=>" => MatchResult::Match(LexMatch::Tok(TokInfo::Impl)),
287            "<=" => MatchResult::Match(LexMatch::Tok(TokInfo::Lte)),
288            "<>" => MatchResult::Match(LexMatch::Tok(TokInfo::Neq)),
289            ">=" => MatchResult::Match(LexMatch::Tok(TokInfo::Gte)),
290            ".." => MatchResult::Match(LexMatch::Tok(TokInfo::CDots)),
291            "**" => MatchResult::Match(LexMatch::Tok(TokInfo::Power)),
292            "<<" => MatchResult::Match(LexMatch::Tok(TokInfo::OpenStaticPar)),
293            ">>" => MatchResult::Match(LexMatch::Tok(TokInfo::CloseStaticPar)),
294            "+" => MatchResult::Match(LexMatch::Tok(TokInfo::Plus)),
295            "^" => MatchResult::Match(LexMatch::Tok(TokInfo::Hat)),
296            "#" => MatchResult::Match(LexMatch::Tok(TokInfo::Sharp)),
297            "-" => MatchResult::Ambiguous(vec![TokInfo::Minus]),
298            "/" => MatchResult::Match(LexMatch::Tok(TokInfo::Slash)),
299            "%" => MatchResult::Match(LexMatch::Tok(TokInfo::Percent)),
300            "*" => MatchResult::Match(LexMatch::Tok(TokInfo::Star)),
301            "|" => MatchResult::Match(LexMatch::Tok(TokInfo::Bar)),
302            "=" => MatchResult::Match(LexMatch::Tok(TokInfo::Equal)),
303            "." => MatchResult::Match(LexMatch::Tok(TokInfo::Dot)),
304            "(" => MatchResult::Match(LexMatch::Tok(TokInfo::OpenPar)),
305            ")" => MatchResult::Match(LexMatch::Tok(TokInfo::ClosePar)),
306            "{" => MatchResult::Match(LexMatch::Tok(TokInfo::OpenBrace)),
307            "}" => MatchResult::Match(LexMatch::Tok(TokInfo::CloseBrace)),
308            "[" => MatchResult::Match(LexMatch::Tok(TokInfo::OpenBracket)),
309            "]" => MatchResult::Match(LexMatch::Tok(TokInfo::CloseBracket)),
310            "<" => MatchResult::Match(LexMatch::Tok(TokInfo::Lt)),
311            ">" => MatchResult::Match(LexMatch::Tok(TokInfo::Gt)),
312            "extern" => MatchResult::Match(LexMatch::Tok(TokInfo::Extern)),
313            "unsafe" => MatchResult::Match(LexMatch::Tok(TokInfo::Unsafe)),
314            "and" => MatchResult::Match(LexMatch::Tok(TokInfo::And)),
315            "assert" => MatchResult::Match(LexMatch::Tok(TokInfo::Assert)),
316            "bool" => MatchResult::Match(LexMatch::Tok(TokInfo::Bool)),
317            "const" => MatchResult::Match(LexMatch::Tok(TokInfo::Const)),
318            "current" => MatchResult::Match(LexMatch::Tok(TokInfo::Current)),
319            "div" => MatchResult::Match(LexMatch::Tok(TokInfo::Div)),
320            "else" => MatchResult::Match(LexMatch::Tok(TokInfo::Else)),
321            "enum" => MatchResult::Match(LexMatch::Tok(TokInfo::Enum)),
322            "function" => MatchResult::Match(LexMatch::Tok(TokInfo::Function)),
323            "false" => MatchResult::Match(LexMatch::Tok(TokInfo::False)),
324            "if" => MatchResult::Match(LexMatch::Tok(TokInfo::If)),
325            "int" => MatchResult::Match(LexMatch::Tok(TokInfo::Int)),
326            "let" => MatchResult::Match(LexMatch::Tok(TokInfo::Let)),
327            "mod" => MatchResult::Match(LexMatch::Tok(TokInfo::Mod)),
328            "node" => MatchResult::Match(LexMatch::Tok(TokInfo::Node)),
329            "not" => MatchResult::Match(LexMatch::Tok(TokInfo::Not)),
330            "operator" => MatchResult::Match(LexMatch::Tok(TokInfo::Operator)),
331            "or" => MatchResult::Match(LexMatch::Tok(TokInfo::Or)),
332            "nor" => MatchResult::Match(LexMatch::Tok(TokInfo::Nor)),
333            "fby" => MatchResult::Match(LexMatch::Tok(TokInfo::FBy)),
334            "pre" => MatchResult::Match(LexMatch::Tok(TokInfo::Pre)),
335            "real" => MatchResult::Match(LexMatch::Tok(TokInfo::Real)),
336            "returns" => MatchResult::Match(LexMatch::Tok(TokInfo::Returns)),
337            "step" => MatchResult::Match(LexMatch::Tok(TokInfo::Step)),
338            "struct" => MatchResult::Match(LexMatch::Tok(TokInfo::Struct)),
339            "tel" => MatchResult::Match(LexMatch::Tok(TokInfo::Tel)),
340            "type" => MatchResult::Match(LexMatch::Tok(TokInfo::Type)),
341            "then" => MatchResult::Match(LexMatch::Tok(TokInfo::Then)),
342            "true" => MatchResult::Match(LexMatch::Tok(TokInfo::True)),
343            "var" => MatchResult::Match(LexMatch::Tok(TokInfo::Var)),
344            "when" => MatchResult::Match(LexMatch::Tok(TokInfo::When)),
345            "with" => MatchResult::Match(LexMatch::Tok(TokInfo::With)),
346            "xor" => MatchResult::Match(LexMatch::Tok(TokInfo::Xor)),
347            "model" => MatchResult::Match(LexMatch::Tok(TokInfo::Model)),
348            "package" => MatchResult::Match(LexMatch::Tok(TokInfo::Package)),
349            "needs" => MatchResult::Match(LexMatch::Tok(TokInfo::Needs)),
350            "provides" => MatchResult::Match(LexMatch::Tok(TokInfo::Provides)),
351            "uses" => MatchResult::Match(LexMatch::Tok(TokInfo::Uses)),
352            "is" => MatchResult::Match(LexMatch::Tok(TokInfo::Is)),
353            "body" => MatchResult::Match(LexMatch::Tok(TokInfo::Body)),
354            "end" => MatchResult::Match(LexMatch::Tok(TokInfo::End)),
355            "include" => MatchResult::Match(LexMatch::Tok(TokInfo::Include)),
356            "merge" => MatchResult::Match(LexMatch::Tok(TokInfo::Merge)),
357            x if x.chars().all(char::is_numeric) => {
358                MatchResult::Match(LexMatch::Tok(TokInfo::IConst(x.parse::<i64>().unwrap())))
359            }
360            x if x.chars().all(|c| {
361                c.is_numeric() || c == '.' || c == 'e' || c == 'E' || c == '+' || c == '-'
362            }) && x.chars().any(char::is_numeric)
363                && (!(x.contains('+') || x.contains('-'))
364                    || x.contains("e-")
365                    || x.contains("e+")
366                    || x.contains("E+")
367                    || x.contains("E-")) =>
368            {
369                // Maybe we are parsing something like `1..2`
370                if x.ends_with('.') {
371                    if x.ends_with("..") {
372                        MatchResult::ManyMatches(vec![
373                            TokInfo::IConst(x[..x.len() - 2].parse::<i64>().unwrap()),
374                            TokInfo::CDots,
375                        ])
376                    } else {
377                        MatchResult::Ambiguous(vec![TokInfo::RConst(x.parse::<f64>().unwrap())])
378                    }
379                } else if x.contains("..") {
380                    MatchResult::NoMatches
381                } else if x.starts_with('e') || x.starts_with('E') {
382                    MatchResult::Match(LexMatch::Tok(TokInfo::Ident(x)))
383                } else if x.ends_with('e') || x.ends_with('E') {
384                    MatchResult::NotYetMatched
385                } else if x.ends_with('-') {
386                    let mut tokens = Vec::with_capacity(2);
387                    match self.match_tokens(&x[..x.len() - 1]) {
388                        MatchResult::Ambiguous(ref mut t) | MatchResult::ManyMatches(ref mut t) => {
389                            tokens.append(t)
390                        }
391                        MatchResult::Match(LexMatch::Tok(m)) => tokens.push(m),
392                        _ => {}
393                    }
394                    tokens.push(TokInfo::Minus);
395                    MatchResult::Ambiguous(tokens)
396                } else if x.ends_with('+') {
397                    let mut tokens = Vec::with_capacity(2);
398                    match self.match_tokens(&x[..x.len() - 1]) {
399                        MatchResult::Ambiguous(ref mut t) | MatchResult::ManyMatches(ref mut t) => {
400                            tokens.append(t)
401                        }
402                        MatchResult::Match(LexMatch::Tok(m)) => tokens.push(m),
403                        _ => {}
404                    }
405                    tokens.push(TokInfo::Plus);
406                    MatchResult::Ambiguous(tokens)
407                } else if x.starts_with('-') {
408                    let mut tokens = Vec::with_capacity(2);
409                    tokens.push(TokInfo::Minus);
410                    match self.match_tokens(&x[1..]) {
411                        MatchResult::Ambiguous(ref mut t) | MatchResult::ManyMatches(ref mut t) => {
412                            tokens.append(t)
413                        }
414                        MatchResult::Match(LexMatch::Tok(m)) => tokens.push(m),
415                        _ => {}
416                    }
417                    MatchResult::ManyMatches(tokens)
418                } else if x.starts_with('+') {
419                    let mut tokens = Vec::with_capacity(2);
420                    tokens.push(TokInfo::Plus);
421                    match self.match_tokens(&x[1..]) {
422                        MatchResult::Ambiguous(ref mut t) | MatchResult::ManyMatches(ref mut t) => {
423                            tokens.append(t)
424                        }
425                        MatchResult::Match(LexMatch::Tok(m)) => tokens.push(m),
426                        _ => {}
427                    }
428                    MatchResult::ManyMatches(tokens)
429                } else {
430                    MatchResult::Match(LexMatch::Tok(TokInfo::RConst(x.parse::<f64>().unwrap())))
431                }
432            }
433            x if x.chars().all(|c| c.is_alphanumeric() || c == '_') => {
434                MatchResult::Match(LexMatch::Tok(TokInfo::Ident(x)))
435            }
436            // yet another special case for real constants
437            // this one is for handling things like: 1->pre x
438            x if x.ends_with("->") => {
439                let mut tokens = Vec::with_capacity(2);
440                match self.match_tokens(&x[..x.len() - 2]) {
441                    MatchResult::Ambiguous(ref mut t) | MatchResult::ManyMatches(ref mut t) => {
442                        tokens.append(t)
443                    }
444                    MatchResult::Match(LexMatch::Tok(m)) => tokens.push(m),
445                    _ => {}
446                }
447                tokens.push(TokInfo::Arrow);
448                MatchResult::ManyMatches(tokens)
449            }
450            _ => MatchResult::NoMatches,
451        }
452    }
453
454    fn span(&self, line: usize, col: usize, pos: usize, len: usize) -> Span {
455        Span {
456            start: Location {
457                line: line as u64,
458                col: col as u64,
459                pos: pos as u64,
460            },
461            end: Location {
462                line: line as u64, // no token can be on multiple lines as far as I know
463                col: (col + len) as u64,
464                pos: (pos + len) as u64,
465            },
466            file: self.file,
467        }
468    }
469}
470
471#[cfg(test)]
472mod tests {
473    use super::*;
474
475    fn check_tok_info<'a, 'f>(actual: Vec<Tok<'a, 'f>>, expected: Vec<TokInfo<'a>>) {
476        dbg!(&actual);
477        assert_eq!(actual.len(), expected.len());
478
479        for (ref a, ref e) in actual.iter().zip(expected.iter()) {
480            assert_eq!(a.item, **e);
481        }
482    }
483
484    fn test_lexer<'a>(src: &'a str, expected: Vec<TokInfo<'a>>) {
485        let mut lex = Lexer::new("main.lus", src);
486        let toks = lex.lex().unwrap();
487        check_tok_info(toks, expected);
488    }
489
490    #[test]
491    fn test_empty() {
492        test_lexer("", vec![TokInfo::EOF])
493    }
494
495    #[test]
496    fn test_keyword() {
497        test_lexer("function", vec![TokInfo::Function, TokInfo::EOF])
498    }
499
500    #[test]
501    fn test_arrow() {
502        test_lexer(
503            "y=0->pre",
504            vec![
505                TokInfo::Ident("y"),
506                TokInfo::Equal,
507                TokInfo::IConst(0),
508                TokInfo::Arrow,
509                TokInfo::Pre,
510                TokInfo::EOF,
511            ],
512        )
513    }
514
515    #[test]
516    fn test_keywords() {
517        test_lexer(
518            "extern function",
519            vec![TokInfo::Extern, TokInfo::Function, TokInfo::EOF],
520        );
521        test_lexer(
522            "functional",
523            vec![TokInfo::Ident("functional"), TokInfo::EOF],
524        );
525    }
526
527    #[test]
528    fn test_spaces() {
529        test_lexer(
530            "extern\n  \t\r\nfunction",
531            vec![TokInfo::Extern, TokInfo::Function, TokInfo::EOF],
532        );
533        test_lexer(
534            "\n  \t\r\nextern function",
535            vec![TokInfo::Extern, TokInfo::Function, TokInfo::EOF],
536        );
537        test_lexer(
538            "extern function\n  \t\r\n",
539            vec![TokInfo::Extern, TokInfo::Function, TokInfo::EOF],
540        );
541    }
542
543    #[test]
544    fn test_iconst() {
545        test_lexer(
546            "42 -12",
547            vec![
548                TokInfo::IConst(42),
549                TokInfo::Minus,
550                TokInfo::IConst(12),
551                TokInfo::EOF,
552            ],
553        )
554    }
555
556    #[test]
557    fn test_rconst() {
558        test_lexer("33.3", vec![TokInfo::RConst(33.3), TokInfo::EOF])
559    }
560
561    #[test]
562    fn test_str() {
563        test_lexer(
564            "include \"memoire.lus\"",
565            vec![TokInfo::Include, TokInfo::Str("memoire.lus"), TokInfo::EOF],
566        );
567    }
568
569    #[test]
570    fn test_comments() {
571        test_lexer(
572            "-- comment\nfunction\nfunction --comment",
573            vec![TokInfo::Function, TokInfo::Function, TokInfo::EOF],
574        );
575        test_lexer(
576            "include (* hello *) extern /* world */ function",
577            vec![
578                TokInfo::Include,
579                TokInfo::Extern,
580                TokInfo::Function,
581                TokInfo::EOF,
582            ],
583        )
584    }
585
586    #[test]
587    fn test_ops() {
588        test_lexer(
589            "12 + 3",
590            vec![
591                TokInfo::IConst(12),
592                TokInfo::Plus,
593                TokInfo::IConst(3),
594                TokInfo::EOF,
595            ],
596        );
597        test_lexer(
598            "42*7",
599            vec![
600                TokInfo::IConst(42),
601                TokInfo::Star,
602                TokInfo::IConst(7),
603                TokInfo::EOF,
604            ],
605        );
606    }
607
608    #[test]
609    fn test_const_slice() {
610        test_lexer(
611            "a[1..2]",
612            vec![
613                TokInfo::Ident("a"),
614                TokInfo::OpenBracket,
615                TokInfo::IConst(1),
616                TokInfo::CDots,
617                TokInfo::IConst(2),
618                TokInfo::CloseBracket,
619                TokInfo::EOF,
620            ],
621        )
622    }
623
624    #[test]
625    fn test_ident() {
626        test_lexer(
627            "a aaaa",
628            vec![TokInfo::Ident("a"), TokInfo::Ident("aaaa"), TokInfo::EOF],
629        )
630    }
631
632    #[test]
633    fn test_static_pars() {
634        test_lexer(
635            "function a<<const n : int>>()",
636            vec![
637                TokInfo::Function,
638                TokInfo::Ident("a"),
639                TokInfo::OpenStaticPar,
640                TokInfo::Const,
641                TokInfo::Ident("n"),
642                TokInfo::Colon,
643                TokInfo::Int,
644                TokInfo::CloseStaticPar,
645                TokInfo::OpenPar,
646                TokInfo::ClosePar,
647                TokInfo::EOF,
648            ],
649        );
650        test_lexer(
651            "x + amaury_n<<n-1>>(x);",
652            vec![
653                TokInfo::Ident("x"),
654                TokInfo::Plus,
655                TokInfo::Ident("amaury_n"),
656                TokInfo::OpenStaticPar,
657                TokInfo::Ident("n"),
658                TokInfo::Minus,
659                TokInfo::IConst(1),
660                TokInfo::CloseStaticPar,
661                TokInfo::OpenPar,
662                TokInfo::Ident("x"),
663                TokInfo::ClosePar,
664                TokInfo::Semicolon,
665                TokInfo::EOF,
666            ],
667        )
668    }
669
670    #[test]
671    fn test_unclosed_str() {
672        let mut lexer = Lexer::new("main.lus", "\"hello ");
673        assert_eq!(lexer.lex(), Err(Error::UnclosedStr));
674    }
675
676    #[test]
677    fn test_unclosed_comment() {
678        let mut lexer = Lexer::new("main.lus", "/* hello ");
679        assert_eq!(lexer.lex(), Err(Error::UnclosedComment));
680    }
681
682    #[test]
683    fn test_amaury() {
684        use TokInfo::*;
685        let amaury1 = r#"y = with n = 0 then 0 else
686         x + amaury_n<<n-1>>(x);"#;
687        test_lexer(
688            amaury1,
689            vec![
690                Ident("y"),
691                Equal,
692                With,
693                Ident("n"),
694                Equal,
695                IConst(0),
696                Then,
697                IConst(0),
698                Else,
699                Ident("x"),
700                Plus,
701                Ident("amaury_n"),
702                OpenStaticPar,
703                Ident("n"),
704                Minus,
705                IConst(1),
706                CloseStaticPar,
707                OpenPar,
708                Ident("x"),
709                ClosePar,
710                Semicolon,
711                EOF,
712            ],
713        );
714
715        let amaury = r#"
716node amaury_n<<const n:int>>(x: int) returns (y:int);
717let
718   y = with n = 0 then 0 else
719         x + amaury_n<<n-1>>(x);
720tel
721
722node amaury = amaury_n<<4>>;
723"#;
724        test_lexer(
725            amaury,
726            vec![
727                Node,
728                Ident("amaury_n"),
729                OpenStaticPar,
730                Const,
731                Ident("n"),
732                Colon,
733                Int,
734                CloseStaticPar,
735                OpenPar,
736                Ident("x"),
737                Colon,
738                Int,
739                ClosePar,
740                Returns,
741                OpenPar,
742                Ident("y"),
743                Colon,
744                Int,
745                ClosePar,
746                Semicolon,
747                Let,
748                Ident("y"),
749                Equal,
750                With,
751                Ident("n"),
752                Equal,
753                IConst(0),
754                Then,
755                IConst(0),
756                Else,
757                Ident("x"),
758                Plus,
759                Ident("amaury_n"),
760                OpenStaticPar,
761                Ident("n"),
762                Minus,
763                IConst(1),
764                CloseStaticPar,
765                OpenPar,
766                Ident("x"),
767                ClosePar,
768                Semicolon,
769                Tel,
770                Node,
771                Ident("amaury"),
772                Equal,
773                Ident("amaury_n"),
774                OpenStaticPar,
775                IConst(4),
776                CloseStaticPar,
777                Semicolon,
778                EOF,
779            ],
780        );
781    }
782}
rustre_parser/lexer.rs

rustre_parser/
lexer.rs
