tergo-tokenizer 0.2.2

R language tokenizer
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37

use crate::tokens::CommentedToken;

/// Wrapper for a buffer of tokens used for pretty printing.
#[derive(Debug)]
pub struct TokensBuffer<'a>(pub &'a [&'a CommentedToken<'a>]);

impl<'a> std::fmt::Display for TokensBuffer<'a> {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        match self.0.split_first() {
            None => Ok(()),
            Some((first, rest)) => {
                f.write_fmt(format_args!("{:?}", first.token))?;
                for token in rest {
                    write!(f, " {:?}", token.token)?;
                }
                Ok(())
            }
        }
    }
}

#[cfg(test)]
mod tests {
    use crate::Token;

    use super::*;

    #[test]
    fn test_displaying_tokens() {
        let tokens = [
            CommentedToken::new(Token::Symbol("a"), 1),
            CommentedToken::new(Token::Symbol("b"), 1),
        ];
        let displayed = format!("{}", TokensBuffer(&[&tokens[0], &tokens[1]]));
        assert_eq!("Symbol(\"a\") Symbol(\"b\")", displayed);
    }
}