gtk_ui/
lexer.rs

1// Crates
2
3use unescape::unescape;
4use std::ops::Range;
5use super::{
6    name_range,
7    start_name_range
8};
9
10// Tokens
11
12#[derive(Debug, Clone)]
13pub enum DefinitionType {
14    InlineProp,
15    InlineArg,
16    ChildProp,
17    ChildArg,
18    Object(String)
19}
20
21#[derive(Debug, Clone)]
22pub enum DirectiveType {
23    Include,
24    Header
25}
26#[derive(Debug, Clone)]
27pub enum TypeIdentifierType {
28    String,
29    Number,
30    Bool
31}
32
33#[derive(Debug, Clone)]
34pub enum IdentifierType {
35    Generic(String),
36    Type(TypeIdentifierType)
37}
38
39#[derive(Debug, Clone)]
40pub enum TokenValue {
41    String(String),             // "mystring"
42    Number(f32),                // 0123456789
43    Bool(i32),                  // true, false
44    Definition(DefinitionType), // @mydefinition
45    Directive(DirectiveType),   // #mydirective
46    Setter(String),             // .mysetter
47    Identifier(IdentifierType), // anything else
48    Comment,                    // //
49    Inherits,                   // ->
50    StartBlock,                 // { 
51    EndBlock,                   // }
52    StartArgList,               // (
53    EndArgList,                 // )
54    ArgListDeliminator,         // ,
55}
56
57#[derive(Debug, Clone)]
58pub struct Token {
59    pub value: TokenValue,
60    pub range: Range<usize>
61}
62
63// Position
64
65impl Token {
66    pub fn to_string(&self) -> &str {
67        match &self.value {
68            TokenValue::String(_) => "string",
69            TokenValue::Number(_) => "number",
70            TokenValue::Bool(_) => "boolean",
71            TokenValue::Definition(_) => "definition",
72            TokenValue::Directive(_) => "directive",
73            TokenValue::Setter(_) => "setter",
74            TokenValue::Identifier(_) => "identifier",
75            TokenValue::StartBlock => "{",
76            TokenValue::EndBlock => "}",
77            TokenValue::StartArgList => "(",
78            TokenValue::EndArgList => ")",
79            TokenValue::ArgListDeliminator => ",",
80            TokenValue::Inherits => "->",
81            TokenValue::Comment => "comment"
82        }
83    }
84    
85    pub fn value_to_string(&self) -> String {
86        match &self.value {
87            TokenValue::String(string) => string.to_string(),
88            TokenValue::Number(number) => number.to_string(),
89            TokenValue::Bool(boolean) => boolean.to_string(),
90            _ => todo!("not implemented yet, but not needed yet!")
91        }
92    }
93}
94
95impl DefinitionType {
96    pub fn from(definition: &String) -> TokenValue {
97        TokenValue::Definition(
98            if definition == "InlineProp" {
99                DefinitionType::InlineProp
100            } else if definition == "InlineArg" {
101                DefinitionType::InlineArg
102            } else if definition == "ChildProp" {
103                DefinitionType::ChildProp
104            } else if definition == "ChildArg" {
105                DefinitionType::ChildArg
106            } else {
107                DefinitionType::Object(String::from(definition))
108            }
109        )
110    }
111}
112
113impl DefinitionType {
114    pub fn to_string(&self) -> &str {
115        match self {
116            DefinitionType::InlineArg => "InlineArg",
117            DefinitionType::InlineProp => "InlineProp",
118            DefinitionType::ChildArg => "ChildArg",
119            DefinitionType::ChildProp => "ChildProp",
120            DefinitionType::Object(_) => "Object"
121        }
122    }
123}
124
125impl DirectiveType {
126    pub fn from(directive: &String) -> TokenValue {
127        let directive_type: Option<DirectiveType>;
128
129        if directive == "include" {
130            directive_type = Some(DirectiveType::Include);
131        } else if directive == "header" {
132            directive_type = Some(DirectiveType::Header);
133        } else {
134            directive_type = None;
135        }
136
137        match directive_type {
138            Some(t) => TokenValue::Directive(t),
139            None => panic!("invalid directive")
140        }
141    }
142}
143
144// Lexer
145
146pub struct Lexer {
147    pub tokens: Vec<Token>,
148    index: usize,
149    input: String,
150}
151
152impl Lexer {
153
154    // Helper Functions
155
156    #[inline]
157    fn move_foward(&mut self) {
158        self.index += 1;
159    }
160
161    #[inline]
162    fn move_forward_n(&mut self, n: usize) {
163        self.index += n;
164    }
165
166    // Lexing Functions
167
168    fn definition(&mut self) -> Result<Token, (String, Range<usize>)> {
169        let mut definition = String::new();
170        let start_position = self.index.clone();
171        self.move_foward();
172        for c in self.input[self.index..].chars() {
173            match c {
174                name_range!() => {
175                    definition.push(c);
176                },
177                _ => break
178            }
179        }
180
181        self.move_forward_n(definition.len());
182        Ok(Token {
183            value: DefinitionType::from(&definition),
184            range: (start_position..self.index)
185        })
186    }
187
188    fn directive(&mut self) -> Result<Token, (String, Range<usize>)> {
189        let mut directive = String::new();
190        let start_position = self.index.clone();
191        self.move_foward();
192        for c in self.input[self.index..].chars() {
193            match c {
194                name_range!() => {
195                    directive.push(c);
196                },
197                _ => break
198            }
199        }
200
201        self.move_forward_n(directive.len());
202        Ok(Token {
203            value: DirectiveType::from(&directive),
204            range: (start_position..self.index)
205        })
206    }
207
208    fn string(&mut self) -> Result<Token, (String, Range<usize>)> {
209        let mut string = String::new();
210        let start_position = self.index.clone();
211        self.move_foward();
212        loop {
213            let c = self.input.chars().nth(self.index).unwrap();
214            match c {
215                '\\' => {
216                    string.push(c);
217                    string.push(self.input.chars().nth(self.index + 1).unwrap());
218                    self.move_forward_n(2);
219                },
220                '"' => {
221                    self.move_foward();
222                    break
223                },
224                '\n' => {
225                    return Err(( String::from("unexpected end of string input"), (self.index..self.index) ));
226                },
227                _ => {
228                    self.move_foward();
229                    string.push(c);
230                }
231            }
232        }
233
234        match unescape(string.as_str()) {
235            Some(string) =>
236                Ok(Token {
237                    value: TokenValue::String(string),
238                    range: (start_position..self.index)
239                }),
240            None => Err(( String::from("unable to escape string"), (start_position..self.index) ))
241        }
242    }
243
244    fn setter(&mut self) -> Result<Token, (String, Range<usize>)> {
245        let mut setter = String::new();
246        let start_position = self.index.clone();
247        self.move_foward();
248        for c in self.input[self.index..].chars() {
249            match c {
250                name_range!() => {
251                    setter.push(c);
252                },
253                _ => break
254            }
255        }
256
257        self.move_forward_n(setter.len());
258        Ok(Token {
259            value: TokenValue::Setter(setter),
260            range: (start_position..self.index)
261        })
262    }
263
264    fn number(&mut self) -> Result<Token, (String, Range<usize>)> {
265        let mut number = String::new();
266        let start_position = self.index.clone();
267        for c in self.input[self.index..].chars() {
268            if ! c.is_digit(10) && c != '.' {
269                break
270            }
271            number.push(c);
272        }
273
274        self.move_forward_n(number.len());
275
276        match number.parse::<f32>() {
277            Ok(num) =>
278                Ok(Token {
279                    value: TokenValue::Number(num),
280                    range: (start_position..self.index)
281                }),
282            Err(e) => Err((e.to_string(), (start_position..self.index)))
283        }
284    }
285
286    // TODO: Rename this function to include its use with parsing booleans
287    fn identifier(&mut self) -> Result<Token, (String, Range<usize>)> {
288        let mut identifier = String::new();
289        let start_position = self.index.clone();
290        for c in self.input[self.index..].chars() {
291            match c {
292                name_range!() => {
293                    identifier.push(c);
294                },
295                _ => break
296            }
297        }
298
299        self.move_forward_n(identifier.len());
300
301        let value = match identifier.as_str() {
302            "true"   => TokenValue::Bool(1),
303            "false"  => TokenValue::Bool(0),
304            "String" => TokenValue::Identifier(IdentifierType::Type(TypeIdentifierType::String)),
305            "Number" => TokenValue::Identifier(IdentifierType::Type(TypeIdentifierType::Number)),
306            "Bool"   => TokenValue::Identifier(IdentifierType::Type(TypeIdentifierType::Bool)),
307            _        => TokenValue::Identifier(IdentifierType::Generic(identifier))
308        };
309
310        Ok(Token {
311            value,
312            range: (start_position..self.index)
313        })
314    }
315
316    fn add_and_move(&mut self, value: TokenValue) -> Result<Token, (String, Range<usize>)> {
317        self.move_foward();
318        Ok(Token {
319            value,
320            range: ((self.index - 1)..self.index)
321        })
322    }
323
324    fn comment(&mut self) -> Result<Token, (String, Range<usize>)> {
325        let start_position = self.index.clone();
326        loop {
327            let c = self.input.chars().nth(self.index);
328            if let Some(c) = c {
329                if c == '\n' {
330                    break;
331                }
332                self.move_foward();
333            } else {
334                break;
335            }
336        }
337        return Ok(Token {
338            value: TokenValue::Comment,
339            range: (start_position..self.index)
340        })
341    }
342
343    fn inherits(&mut self) -> Result<Token, (String, Range<usize>)> {
344        self.move_foward();
345        if let Some(char) = self.input.chars().nth(self.index) {
346            if char == '>' {
347                self.move_foward();
348                Ok(Token {
349                    value: TokenValue::Inherits,
350                    range: ((self.index - 2)..self.index)
351                })
352            } else {
353                Err((format!("unrecognized character '{}'", char), (self.index..(self.index + 1))))
354            }
355        } else {
356            Err((format!("unexpected end of input"), (self.index..self.index)))
357        }
358    }
359
360    // Pubs
361    pub fn new(s: String) -> Self {
362        Self {
363            tokens: Vec::new(),
364            index: 0,
365            input: s,
366        }
367    }
368    pub fn lex(&mut self, lex_comments: bool) -> Result<(), (String, Range<usize>)> {
369        loop {
370            let input_char = self.input.chars().nth(self.index);
371            if let Some(c) = input_char {
372                let token = match c {
373                    '@'                 => self.definition(),
374                    '#'                 => self.directive(),
375                    '"'                 => self.string(),
376                    '.'                 => self.setter(),
377                    '0'..='9'           => self.number(),
378                    '-'                 => self.inherits(),
379                    start_name_range!() => self.identifier(),
380                    '{'                 => self.add_and_move(TokenValue::StartBlock),
381                    '}'                 => self.add_and_move(TokenValue::EndBlock),
382                    ','                 => self.add_and_move(TokenValue::ArgListDeliminator),
383                    '('                 => self.add_and_move(TokenValue::StartArgList),
384                    ')'                 => self.add_and_move(TokenValue::EndArgList),
385                    ' ' | '\t' | '\n'   => {
386                        self.move_foward();
387                        continue
388                    },
389                    '/'                 => {
390                        let comment = self.comment();
391                        if lex_comments {
392                            comment
393                        } else {
394                            continue
395                        }
396                    },
397                    _ => Err((format!("unrecognized character '{}'", c), (self.index..self.index))),
398                };
399
400                match token {
401                    Ok(token) => self.tokens.push(token),
402                    Err(err) => return Err(err)
403                }
404            } else {
405                return Ok(());
406            }
407        }
408    }
409}
gtk_ui/lexer.rs

gtk_ui/
lexer.rs
