tergo_tokenizer/
tokens.rs

1/// Representation of a token.
2///
3/// This represents a single token in an R program along with the line on which it occurs
4/// and the column offset. Additionally, it stores the comments that are associated
5/// with the token.
6#[derive(Debug, Clone)]
7pub struct CommentedToken<'a> {
8    /// The actual token stored in this struct.
9    pub token: Token<'a>,
10    /// The column offset of the start of this token.
11    pub offset: usize,
12    /// Preceding comments.
13    pub leading_comments: Option<Vec<&'a str>>,
14    /// Trailing inline comment.
15    pub inline_comment: Option<&'a str>,
16}
17
18impl<'a> Deref for CommentedToken<'a> {
19    type Target = Token<'a>;
20
21    fn deref(&self) -> &Self::Target {
22        &self.token
23    }
24}
25
26impl<'a> CommentedToken<'a> {
27    pub fn new(token: Token<'a>, offset: usize) -> Self {
28        Self {
29            token,
30            offset,
31            leading_comments: None,
32            inline_comment: None,
33        }
34    }
35
36    pub fn with_comments(
37        token: Token<'a>,
38        offset: usize,
39        leading_comments: Option<Vec<&'a str>>,
40        inline_comment: Option<&'a str>,
41    ) -> Self {
42        Self {
43            token,
44            offset,
45            leading_comments,
46            inline_comment,
47        }
48    }
49}
50
51/// When comparing two tokens, only the token itself is compared.
52/// The line and offset are ignored.
53impl PartialEq for CommentedToken<'_> {
54    fn eq(&self, other: &Self) -> bool {
55        self.token == other.token
56    }
57}
58
59impl std::fmt::Display for CommentedToken<'_> {
60    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
61        f.write_fmt(format_args!("{:?}", self.token))
62    }
63}
64
65/// This represents all the different token types encountered
66/// in an R program.
67#[derive(Debug, Clone, PartialEq)]
68pub enum Token<'a> {
69    Symbol(&'a str),
70    Literal(&'a str),
71    Semicolon,
72    Newline,
73    LParen,
74    RParen,
75    LBrace,
76    RBrace,
77    LBracket,
78    RBracket,
79    Comma,
80
81    // Reserved
82    Continue,
83    Break,
84    Stop,
85
86    // Compound
87    If,
88    Else,
89    While,
90    For,
91    Repeat,
92    In,
93    Function,
94    Lambda,
95
96    // Binary operators
97    LAssign,
98    SuperAssign,
99    ColonAssign,
100    RAssign,
101    OldAssign,
102    Equal,
103    NotEqual,
104    LowerThan,
105    GreaterThan,
106    LowerEqual,
107    GreaterEqual,
108    Power,
109    Divide,
110    Multiply,
111    Minus,
112    Plus,
113    Help,
114    And,
115    VectorizedAnd,
116    Or,
117    VectorizedOr,
118    Dollar,
119    Pipe,
120    Modulo,
121    NsGet,
122    NsGetInt,
123    Tilde,
124    Colon,
125    Slot,
126    Special(&'a str),
127
128    // Unary operators
129    UnaryNot,
130
131    // Comments
132    InlineComment(&'a str),
133    Comment(&'a str),
134
135    // EOF
136    EOF,
137}
138
139#[macro_export]
140macro_rules! commented_tokens {
141    ($($args:expr),*) => {{
142        vec![
143        $(
144            CommentedToken::new($args, 0),
145        )*
146        ]
147    }}
148}
149use std::ops::Deref;
150
151pub use commented_tokens;
152
153#[cfg(test)]
154mod tests {
155    use super::*;
156    use crate::Token::*;
157
158    #[test]
159    fn commented_tokens_macro() {
160        let tokens = commented_tokens![Symbol("a"), InlineComment("# Comment")];
161        assert_eq!(tokens.len(), 2);
162        assert_eq!(tokens[0].token, Symbol("a"));
163        assert_eq!(tokens[1].token, InlineComment("# Comment"));
164    }
165
166    #[test]
167    fn test_display() {
168        let token = CommentedToken::new(Token::Symbol("a"), 0);
169        let displayed = format!("{}", token);
170        assert_eq!("Symbol(\"a\")", displayed);
171    }
172}
tergo_tokenizer/tokens.rs

tergo_tokenizer/
tokens.rs
