fluid_parser/
lexer.rs

1use crate::error::Location;
2use crate::token::{Token, TokenType};
3
4pub struct Lexer<'a> {
5    pub s: &'a str,
6    size: usize,
7    cursor: usize,
8    line: usize,
9    col: usize,
10}
11
12impl<'a> Lexer<'a> {
13    pub fn new(source: &str) -> Lexer {
14        let size = source.len();
15        let mut cursor = 0;
16        let mut line = 1;
17        let mut col = 1;
18        if source.starts_with("# ") {
19            while cursor < size && source.as_bytes()[cursor] != b'\n' {
20                cursor += 1;
21                col += 1;
22            }
23            if cursor < size && source.as_bytes()[cursor] == b'\n' {
24                cursor += 1;
25                line += 1;
26                col = 1;
27            }
28        }
29        Lexer {
30            s: source,
31            size,
32            cursor,
33            line,
34            col,
35        }
36    }
37
38    #[allow(clippy::should_implement_trait)]
39    pub fn next(&mut self) -> Token<'a> {
40        let mut t = Token::<'_> {
41            loc: Location {
42                line: self.line,
43                col: self.col,
44            },
45            ..Default::default()
46        };
47        if self.cursor == self.size {
48            t.typ = TokenType::Eof;
49            t.start = self.size;
50            t.end = self.size;
51            return t;
52        }
53        while self.cursor < self.size
54            && (self.s.as_bytes()[self.cursor] == b' '
55                || self.s.as_bytes()[self.cursor] == b'\n'
56                || self.s.as_bytes()[self.cursor] == b'\r'
57                || self.s.as_bytes()[self.cursor] == b'\t')
58        {
59            if self.s.as_bytes()[self.cursor] == b'\n' {
60                self.line += 1;
61                self.col = 1;
62            } else {
63                self.col += 1;
64            }
65            self.cursor += 1;
66        }
67        if self.cursor == self.size {
68            t.typ = TokenType::Eof;
69            t.start = self.size;
70            t.end = self.size;
71            return t;
72        }
73        t.loc = Location {
74            line: self.line,
75            col: self.col,
76        };
77        match self.s.as_bytes()[self.cursor] {
78            b'{' => {
79                t.typ = TokenType::OpenBrace;
80                t.start = self.cursor;
81                t.end = self.cursor + 1;
82                self.cursor += 1;
83                self.col += 1;
84                t
85            }
86            b'}' => {
87                t.typ = TokenType::CloseBrace;
88                t.start = self.cursor;
89                t.end = self.cursor + 1;
90                self.cursor += 1;
91                self.col += 1;
92                t
93            }
94            _ => {
95                t.typ = TokenType::Word;
96
97                t.start = self.cursor;
98                while self.cursor < self.size
99                    && self.s.as_bytes()[self.cursor] != b' '
100                    && self.s.as_bytes()[self.cursor] != b'\n'
101                    && self.s.as_bytes()[self.cursor] != b'\r'
102                    && self.s.as_bytes()[self.cursor] != b'\t'
103                    && self.s.as_bytes()[self.cursor] != b'}'
104                {
105                    self.cursor += 1;
106                    self.col += 1;
107                }
108                t.end = self.cursor;
109                t.word = &self.s[t.start..t.end];
110                t
111            }
112        }
113    }
114}
fluid_parser/lexer.rs

fluid_parser/
lexer.rs
