tergo_tokenizer/
tokenizer.rs

1use std::str::CharIndices;
2
3use log::trace;
4
5use crate::tokens::{
6    CommentedToken,
7    Token::{self, *},
8};
9
10/// Tokenizer for an R program.
11///
12/// Transforms an R program into an array of language tokens.
13pub struct Tokenizer<'a> {
14    offset: usize,
15    it: usize,
16    current_char: char,
17    source: CharIndices<'a>,
18    raw_source: &'a str,
19}
20
21const SYMBOL_ENDING: [char; 29] = [
22    ' ', '(', ')', '{', '}', '#', ';', '\n', '\t', '\r', '+', '-', '/', '\\', '%', '*', '^', '!',
23    '&', '|', '<', '>', '=', ',', '[', ']', '$', '`', '"',
24];
25impl<'a> Tokenizer<'a> {
26    /// Returns a new tokenizer from an R program.
27    ///
28    /// # Arguments
29    ///
30    /// * `input` - the R program to tokenize
31    ///
32    /// # Examples
33    ///
34    /// ```
35    /// use tergo_tokenizer::tokenizer::Tokenizer;
36    ///
37    /// let r_program = r#"
38    /// a <- 7
39    /// print(a + 1)
40    /// "#;
41    /// let mut tokenizer = Tokenizer::new(r_program);
42    /// let tokens = tokenizer.tokenize();
43    /// println!("{tokens:?}");
44    /// ```
45    ///
46    pub fn new(input: &'a str) -> Self {
47        Self {
48            offset: 0,
49            it: 0,
50            current_char: '\0',
51            source: input.char_indices(),
52            raw_source: input,
53        }
54    }
55
56    /// Returns an array of tokens.
57    ///
58    /// # Examples
59    ///
60    /// ```
61    /// use tergo_tokenizer::tokenizer::Tokenizer;
62    ///
63    /// let r_program = r#"
64    /// a <- 7
65    /// print(a + 1)
66    /// "#;
67    /// let mut tokenizer = Tokenizer::new(r_program);
68    /// let tokens = tokenizer.tokenize();
69    /// println!("{tokens:?}");
70    /// ```
71    ///
72    pub fn tokenize(&mut self) -> Result<Vec<CommentedToken>, crate::Error> {
73        let mut tokens = vec![];
74        self.next();
75        while self.it < self.raw_source.len() {
76            match self.current_char {
77                ' ' | '\t' => {
78                    self.next();
79                }
80                '\r' => {
81                    self.next();
82                    self.push_token(Newline, &mut tokens);
83                    self.next();
84                }
85                '\n' => {
86                    self.push_token(Newline, &mut tokens);
87                    self.next();
88                }
89                ';' => {
90                    self.push_token(Semicolon, &mut tokens);
91                    self.next();
92                }
93                ',' => {
94                    self.push_token(Comma, &mut tokens);
95                    self.next();
96                }
97                '(' => {
98                    self.push_token(LParen, &mut tokens);
99                    self.next();
100                }
101                ')' => {
102                    self.push_token(RParen, &mut tokens);
103                    self.next();
104                }
105                '{' => {
106                    self.push_token(LBrace, &mut tokens);
107                    self.next();
108                }
109                '}' => {
110                    self.push_token(RBrace, &mut tokens);
111                    self.next();
112                }
113                '[' => {
114                    self.push_token(LBracket, &mut tokens);
115                    self.next();
116                }
117                ']' => {
118                    self.push_token(RBracket, &mut tokens);
119                    self.next();
120                }
121                '\'' | '\"' => {
122                    self.string_literal(&mut tokens);
123                    self.next();
124                }
125                '*' => {
126                    let next_char = self.lookahead().expect("Script does not end on '*'");
127                    match next_char {
128                        // That's undocumented, but it actually works...
129                        '*' => {
130                            self.push_token(Power, &mut tokens);
131                            self.next();
132                        }
133                        _ => self.push_token(Multiply, &mut tokens),
134                    }
135                    self.next();
136                }
137                '/' => {
138                    self.push_token(Divide, &mut tokens);
139                    self.next();
140                }
141                '^' => {
142                    self.push_token(Power, &mut tokens);
143                    self.next();
144                }
145                '+' => {
146                    self.push_token(Plus, &mut tokens);
147                    self.next();
148                }
149                '?' => {
150                    self.push_token(Help, &mut tokens);
151                    self.next();
152                }
153                '<' => {
154                    let next_char = self.lookahead().expect("Script does not end on '<'");
155                    match next_char {
156                        '-' => {
157                            self.push_token(LAssign, &mut tokens);
158                            self.next();
159                        }
160                        '=' => {
161                            self.push_token(LowerEqual, &mut tokens);
162                            self.next();
163                        }
164                        '<' => {
165                            self.push_token(SuperAssign, &mut tokens);
166                            self.next();
167                            self.next();
168                        }
169                        _ => self.push_token(LowerThan, &mut tokens),
170                    }
171                    self.next();
172                }
173                '>' => {
174                    let next_char = self.lookahead().expect("Script does not end on '>'");
175                    match next_char {
176                        '=' => {
177                            self.push_token(GreaterEqual, &mut tokens);
178                            self.next();
179                        }
180                        _ => {
181                            self.push_token(GreaterThan, &mut tokens);
182                        }
183                    }
184                    self.next();
185                }
186                '|' => {
187                    let next_char = self.lookahead().expect("Script does not end on '|'");
188                    match next_char {
189                        '|' => {
190                            self.push_token(Or, &mut tokens);
191                            self.next();
192                        }
193                        '>' => {
194                            self.push_token(Pipe, &mut tokens);
195                            self.next();
196                        }
197                        _ => self.push_token(VectorizedOr, &mut tokens),
198                    }
199                    self.next();
200                }
201                '&' => {
202                    let next_char = self.lookahead().expect("Script does not end on '&'");
203                    match next_char {
204                        '&' => {
205                            self.push_token(And, &mut tokens);
206                            self.next();
207                        }
208                        _ => self.push_token(VectorizedAnd, &mut tokens),
209                    }
210                    self.next();
211                }
212                '=' => {
213                    let next_char = self.lookahead().expect("Script does not end on '='");
214                    match next_char {
215                        '=' => {
216                            self.push_token(Equal, &mut tokens);
217                            self.next();
218                        }
219                        _ => self.push_token(OldAssign, &mut tokens),
220                    }
221                    self.next();
222                }
223                '$' => {
224                    self.push_token(Dollar, &mut tokens);
225                    self.next();
226                }
227                '-' => {
228                    let next_char = self.lookahead().expect("Script does not end on '-'");
229                    match next_char {
230                        '>' => {
231                            self.push_token(RAssign, &mut tokens);
232                            self.next();
233                        }
234                        _ => self.push_token(Minus, &mut tokens),
235                    }
236                    self.next();
237                }
238                '!' => {
239                    self.next();
240                    match self.current_char {
241                        '=' => {
242                            self.push_token(NotEqual, &mut tokens);
243                            self.next();
244                        }
245                        _ => self.push_token(UnaryNot, &mut tokens),
246                    }
247                }
248                '.' => {
249                    let next_char = self.lookahead().expect("Script does not end on '.'");
250                    match next_char {
251                        'a'..='z' | 'A'..='Z' => {
252                            self.identifier(&mut tokens);
253                        }
254                        '0'..='9' => {
255                            self.number_literal(&mut tokens);
256                        }
257                        _ => {
258                            trace!(
259                                "Found not alphabetic and non-numeric character after a dot. \
260                                 Treating it as an identifier."
261                            );
262                            self.identifier(&mut tokens);
263                        }
264                    }
265                }
266                '`' | '_' => {
267                    self.identifier(&mut tokens);
268                }
269                '%' => {
270                    let next_char = self.lookahead().expect("Script does not end on '%'");
271                    match next_char {
272                        '%' => {
273                            self.push_token(Modulo, &mut tokens);
274                            self.next();
275                            self.next();
276                        }
277                        _ => {
278                            let custom_binary_start = self.it;
279                            self.next();
280                            while self.current_char != '%' {
281                                self.next();
282                            }
283                            let custom_binary_end = self.it;
284                            self.push_token(
285                                Special(&self.raw_source[custom_binary_start..=custom_binary_end]),
286                                &mut tokens,
287                            );
288                            self.next()
289                        }
290                    }
291                }
292                'a'..='z' | 'A'..='Z' => match self.lookahead() {
293                    Some('"') | Some('\'') => {
294                        self.raw_string_literal(&mut tokens);
295                    }
296                    Some(_) | None => self.identifier_or_reserved(&mut tokens),
297                },
298                '0'..='9' => {
299                    self.number_literal(&mut tokens);
300                }
301                '\\' => {
302                    self.push_token(Lambda, &mut tokens);
303                    self.next();
304                }
305                '#' => {
306                    self.comment(&mut tokens);
307                }
308                '~' => {
309                    self.push_token(Tilde, &mut tokens);
310                    self.next();
311                }
312                '@' => {
313                    self.push_token(Slot, &mut tokens);
314                    self.next();
315                }
316                ':' => {
317                    self.next();
318                    let next = self.lookahead();
319
320                    match (self.current_char, next) {
321                        // :::
322                        (':', Some(':')) => {
323                            self.push_token(NsGetInt, &mut tokens);
324                            self.next();
325                            self.next();
326                        }
327                        // ::
328                        (':', _) => {
329                            self.push_token(NsGet, &mut tokens);
330                            self.next()
331                        }
332                        // :=
333                        ('=', _) => {
334                            self.push_token(ColonAssign, &mut tokens);
335                            self.next()
336                        }
337                        // :
338                        _ => self.push_token(Colon, &mut tokens),
339                    }
340                }
341                c => {
342                    trace!(
343                        "Unexpected character '{c}' around:\n\"\"\"\n{}\n\"\"\"",
344                        &self.raw_source[self.it.saturating_sub(15)
345                            ..self.it.saturating_add(15).min(self.raw_source.len())]
346                    );
347                    self.next();
348                }
349            }
350        }
351        tokens.push(CommentedToken::new(EOF, self.offset));
352        trace!("Tokenized: {:?}", tokens);
353        Ok(tokens)
354    }
355
356    fn push_token(&mut self, token: Token<'a>, tokens: &mut Vec<CommentedToken<'a>>) {
357        tokens.push(CommentedToken::new(token, self.offset));
358    }
359
360    fn raw_string_literal(&mut self, tokens: &mut Vec<CommentedToken<'a>>) {
361        let start_offset = self.offset;
362        let start_it = self.it;
363        // Skip the r in r"..."
364        self.next();
365
366        // Get the delimiter
367        let delimiter = self.current_char;
368        self.next();
369
370        // Find the inside delimiter
371        const START_DELIMS: [char; 3] = ['(', '{', '['];
372        const END_DELIMS: [char; 3] = [')', '}', ']'];
373        let mut end = vec![];
374        while !START_DELIMS.contains(&self.current_char) {
375            end.push(self.current_char);
376            self.next();
377        }
378
379        end.push(
380            END_DELIMS[START_DELIMS
381                .into_iter()
382                .position(|c| c == self.current_char)
383                .expect("We just checked that START_DELIMS contains the current character")],
384        );
385        end.reverse();
386        end.push(delimiter);
387
388        // Move past the start delimiter
389        self.next();
390
391        // Advance until we find the end delimiter
392        let mut matching_end_chars = 0;
393        while matching_end_chars < end.len() {
394            if end[matching_end_chars] == self.current_char {
395                matching_end_chars += 1;
396            }
397            self.next();
398        }
399
400        // Until self.it because the loop above moves one character past the end delimiter
401        // (including ' or ")
402        tokens.push(CommentedToken::new(
403            Literal(&self.raw_source[start_it..self.it]),
404            start_offset,
405        ));
406    }
407
408    fn string_literal(&mut self, tokens: &mut Vec<CommentedToken<'a>>) {
409        let start_offset = self.offset;
410        let start_it = self.it;
411        let delimiter = self.current_char;
412        let mut in_escape = false;
413        self.next();
414        while self.current_char != delimiter || in_escape {
415            if in_escape {
416                in_escape = !in_escape;
417            } else if self.current_char == '\\' {
418                in_escape = true;
419            }
420            self.next()
421        }
422        tokens.push(CommentedToken::new(
423            Literal(&self.raw_source[start_it..=self.it]),
424            start_offset,
425        ));
426    }
427
428    fn parse_decimal(&mut self) {
429        while self.it < self.raw_source.len() && self.current_char.is_ascii_digit() {
430            self.next();
431        }
432    }
433
434    fn parse_hexadecimal(&mut self) {
435        while self.it < self.raw_source.len() && self.current_char.is_ascii_hexdigit() {
436            self.next();
437        }
438    }
439
440    fn number_literal(&mut self, tokens: &mut Vec<CommentedToken<'a>>) {
441        let start_it = self.it;
442        let next = self.lookahead();
443        match (self.current_char, next) {
444            // Hexadecimal
445            // 0x.., 0X..
446            ('0', Some(next)) if next == 'x' || next == 'X' => {
447                self.next();
448                self.next();
449                self.parse_hexadecimal();
450                if self.current_char == '.' {
451                    self.next();
452                    self.parse_hexadecimal();
453                    if self.current_char == 'p' || self.current_char == 'P' {
454                        self.next();
455                        self.parse_hexadecimal();
456                    }
457                }
458            }
459            // Decimal
460            _ => {
461                self.parse_decimal();
462                let next = self.lookahead();
463                match (self.current_char, next) {
464                    ('.', _) => {
465                        self.next();
466                        self.parse_decimal();
467                        let next = self.lookahead();
468                        match (self.current_char, next) {
469                            ('e', Some(next)) if next == '+' || next == '-' => {
470                                self.next();
471                                self.next();
472                                self.parse_decimal();
473                            }
474                            ('E', Some(next)) if next == '+' || next == '-' => {
475                                self.next();
476                                self.next();
477                                self.parse_decimal();
478                            }
479                            ('e', _) | ('E', _) => {
480                                self.next();
481                                self.parse_decimal();
482                            }
483                            ('i', _) => {
484                                self.next();
485                            }
486                            _ => {}
487                        }
488                    }
489                    ('e', Some(next)) if next == '+' || next == '-' => {
490                        self.next();
491                        self.next();
492                        self.parse_decimal();
493                    }
494                    ('E', Some(next)) if next == '+' || next == '-' => {
495                        self.next();
496                        self.next();
497                        self.parse_decimal();
498                    }
499                    ('e', _) | ('E', _) => {
500                        self.next();
501                        self.parse_decimal();
502                    }
503                    ('L', _) => {
504                        self.next();
505                    }
506                    ('i', _) => {
507                        self.next();
508                    }
509                    _ => {}
510                }
511            }
512        }
513        self.push_token(Literal(&self.raw_source[start_it..self.it]), tokens);
514    }
515
516    fn identifier(&mut self, tokens: &mut Vec<CommentedToken<'a>>) {
517        let start_it = self.it;
518        let mut in_backticks = false;
519        let mut escaped = false;
520        while self.it < self.raw_source.len() && in_backticks
521            || self.current_char.is_alphabetic()
522            || self.current_char.is_ascii_digit()
523            || self.current_char == '.'
524            || self.current_char == '_'
525            || self.current_char == '`'
526        {
527            match self.current_char {
528                '\\' => {
529                    escaped = true;
530                    self.next();
531                }
532                '`' if escaped => {
533                    escaped = false;
534                    self.next();
535                }
536                '`' => {
537                    in_backticks = !in_backticks;
538                    self.next();
539                }
540                _ => self.next(),
541            }
542        }
543        match &self.raw_source[start_it..self.it] {
544            "TRUE" | "T" => self.push_token(Literal("TRUE"), tokens),
545            "FALSE" | "F" => self.push_token(Literal("FALSE"), tokens),
546            _ => self.push_token(Symbol(&self.raw_source[start_it..self.it]), tokens),
547        }
548    }
549
550    fn identifier_or_reserved(&mut self, tokens: &mut Vec<CommentedToken<'a>>) {
551        let start_it = self.it;
552        while self.it < self.raw_source.len() && !SYMBOL_ENDING.contains(&self.current_char) {
553            self.next();
554        }
555
556        match &self.raw_source[start_it..self.it] {
557            "continue" => self.push_token(Continue, tokens),
558            "break" => self.push_token(Break, tokens),
559            "for" => self.push_token(For, tokens),
560            "if" => self.push_token(If, tokens),
561            "else" => self.push_token(Else, tokens),
562            "in" => self.push_token(In, tokens),
563            "while" => self.push_token(While, tokens),
564            "repeat" => self.push_token(Repeat, tokens),
565            "function" => self.push_token(Function, tokens),
566            "TRUE" | "T" => self.push_token(Literal("TRUE"), tokens),
567            "FALSE" | "F" => self.push_token(Literal("FALSE"), tokens),
568            _ => self.push_token(Symbol(&self.raw_source[start_it..self.it]), tokens),
569        }
570    }
571
572    fn comment(&mut self, tokens: &mut Vec<CommentedToken<'a>>) {
573        let start_it = self.it;
574        while self.it < self.raw_source.len() && self.current_char != '\n' {
575            self.next();
576        }
577
578        match tokens.last() {
579            Some(CommentedToken {
580                token: Newline,
581                offset: _,
582                leading_comments: _,
583                inline_comment: _,
584            }) => self.push_token(Comment(&self.raw_source[start_it..self.it]), tokens),
585            Some(_) => self.push_token(InlineComment(&self.raw_source[start_it..self.it]), tokens),
586            None => self.push_token(Comment(&self.raw_source[start_it..self.it]), tokens),
587        }
588    }
589
590    fn next(&mut self) {
591        if let Some((new_offset, new_char)) = self.source.next() {
592            self.offset = new_offset;
593            self.it = new_offset;
594            self.current_char = new_char;
595        } else {
596            self.offset = self.raw_source.len();
597            self.it = self.raw_source.len();
598        }
599    }
600
601    fn lookahead(&self) -> Option<char> {
602        self.source
603            .clone()
604            .peekable()
605            .next()
606            .map(|(_, new_char)| new_char)
607    }
608}
tergo_tokenizer/tokenizer.rs

tergo_tokenizer/
tokenizer.rs
