thrift_analyzer/analyzer/
token.rs

1use std::fmt::{Display, Formatter, Result};
2
3use crate::analyzer::base::{Position, Range};
4
5/// Represents a Thrift token in a document.
6#[derive(PartialEq, Debug, Clone, Default)]
7pub struct Token {
8    pub kind: TokenKind,
9    pub position: Position,
10}
11
12impl Token {
13    /// Returns true if the token is an EOF token.
14    pub fn is_eof(&self) -> bool {
15        self.kind == TokenKind::Eof
16    }
17
18    /// Returns true if the token is an invalid token.
19    pub fn is_invalid(&self) -> bool {
20        match self.kind {
21            TokenKind::Invalid(_) => true,
22            TokenKind::InvalidString(_) => true,
23            _ => false,
24        }
25    }
26
27    /// Returns true if the token is a comment.
28    pub fn is_comment(&self) -> bool {
29        match self.kind {
30            TokenKind::Comment(_) => true,
31            TokenKind::BlockComment(_) => true,
32            TokenKind::PoundComment(_) => true,
33            _ => false,
34        }
35    }
36
37    /// Returns true if the token is a separator.
38    pub fn is_line_separator(&self) -> bool {
39        match self.kind {
40            TokenKind::ListSeparator(_) => true,
41            _ => false,
42        }
43    }
44
45    /// Returns the range of the token.
46    pub fn range(&self) -> Range {
47        let mut end = self.position;
48        end.column += self.kind.len() as u32;
49        Range {
50            start: self.position,
51            end,
52        }
53    }
54}
55
56/// Represents the kind of a Thrift token.
57#[derive(Debug, Clone, PartialEq)]
58pub enum TokenKind {
59    // keywords
60    Include,    // include
61    CppInclude, // cpp_include
62    Namespace,  // namespace
63    Const,      // const
64    Typedef,    // typedef
65    Enum,       // enum
66    Struct,     // struct
67    Union,      // union
68    Exception,  // exception
69    Service,    // service
70    Required,   // required
71    Optional,   // optional
72    Oneway,     // oneway
73    Void,       // void
74    Throws,     // throws
75    Extends,    // extends
76
77    // types
78    Map,     // map
79    Set,     // set
80    List,    // list
81    CppType, // cpp_type
82
83    // characters
84    Assign,  // =
85    Colon,   // :
86    Less,    // <
87    Greater, // >
88    Lparen,  // (
89    Rparen,  // )
90    Lbrace,  // {
91    Rbrace,  // }
92    Lbrack,  // [
93    Rbrack,  // ]
94
95    // comments
96    Comment(String),      // // comment
97    BlockComment(String), // /* comment */
98    PoundComment(String), // # comment
99
100    // constants
101    IntConstant(String),    // 123
102    DoubleConstant(String), // 123.456e-78
103
104    // multi-character tokens
105    NamespaceScope(String), // cpp, go, java, etc.
106    BaseType(String),       // bool, i16, i32, etc.
107    Literal(String),        // literal
108    Identifier(String),     // identifier
109    ListSeparator(char),    // , | ;
110
111    // invalid tokens
112    Invalid(char),
113    InvalidString(String),
114
115    // end of file
116    Eof,
117}
118
119impl Default for TokenKind {
120    fn default() -> Self {
121        TokenKind::Eof
122    }
123}
124
125impl Display for TokenKind {
126    fn fmt(&self, f: &mut Formatter<'_>) -> Result {
127        match self {
128            TokenKind::Include => write!(f, "include"),
129            TokenKind::CppInclude => write!(f, "cpp_include"),
130            TokenKind::Namespace => write!(f, "namespace"),
131            TokenKind::Const => write!(f, "const"),
132            TokenKind::Typedef => write!(f, "typedef"),
133            TokenKind::Enum => write!(f, "enum"),
134            TokenKind::Struct => write!(f, "struct"),
135            TokenKind::Union => write!(f, "union"),
136            TokenKind::Exception => write!(f, "exception"),
137            TokenKind::Service => write!(f, "service"),
138            TokenKind::Required => write!(f, "required"),
139            TokenKind::Optional => write!(f, "optional"),
140            TokenKind::Oneway => write!(f, "oneway"),
141            TokenKind::Void => write!(f, "void"),
142            TokenKind::Throws => write!(f, "throws"),
143            TokenKind::Extends => write!(f, "extends"),
144            TokenKind::Map => write!(f, "map"),
145            TokenKind::Set => write!(f, "set"),
146            TokenKind::List => write!(f, "list"),
147            TokenKind::CppType => write!(f, "cpp_type"),
148            TokenKind::Assign => write!(f, "="),
149            TokenKind::Colon => write!(f, ":"),
150            TokenKind::Less => write!(f, "<"),
151            TokenKind::Greater => write!(f, ">"),
152            TokenKind::Lparen => write!(f, "("),
153            TokenKind::Rparen => write!(f, ")"),
154            TokenKind::Lbrace => write!(f, "{{"),
155            TokenKind::Rbrace => write!(f, "}}"),
156            TokenKind::Lbrack => write!(f, "["),
157            TokenKind::Rbrack => write!(f, "]"),
158            TokenKind::Comment(ref s) => write!(f, "//{}", s),
159            TokenKind::BlockComment(ref s) => write!(f, "/*{}*/", s),
160            TokenKind::PoundComment(ref s) => write!(f, "#{}", s),
161            TokenKind::IntConstant(ref s) => write!(f, "{}", s),
162            TokenKind::DoubleConstant(ref s) => write!(f, "{}", s),
163            TokenKind::NamespaceScope(ref s) => write!(f, "{}", s),
164            TokenKind::BaseType(ref s) => write!(f, "{}", s),
165            TokenKind::Literal(ref s) => write!(f, "{}", s),
166            TokenKind::Identifier(ref s) => write!(f, "{}", s),
167            TokenKind::ListSeparator(c) => write!(f, "{}", c),
168            TokenKind::Invalid(c) => write!(f, "{}", c),
169            TokenKind::InvalidString(ref s) => write!(f, "{}", s),
170            TokenKind::Eof => write!(f, "<EOF>"),
171        }
172    }
173}
174
175impl TokenKind {
176    /// Returns the length of the token.
177    pub fn len(&self) -> usize {
178        match self {
179            TokenKind::Include => 7,
180            TokenKind::CppInclude => 11,
181            TokenKind::Namespace => 9,
182            TokenKind::Const => 5,
183            TokenKind::Typedef => 7,
184            TokenKind::Enum => 4,
185            TokenKind::Struct => 6,
186            TokenKind::Union => 5,
187            TokenKind::Exception => 9,
188            TokenKind::Service => 7,
189            TokenKind::Required => 8,
190            TokenKind::Optional => 8,
191            TokenKind::Oneway => 6,
192            TokenKind::Void => 4,
193            TokenKind::Throws => 6,
194            TokenKind::Extends => 7,
195            TokenKind::Map => 3,
196            TokenKind::Set => 3,
197            TokenKind::List => 4,
198            TokenKind::CppType => 8,
199            TokenKind::Assign => 1,
200            TokenKind::Colon => 1,
201            TokenKind::Less => 1,
202            TokenKind::Greater => 1,
203            TokenKind::Lparen => 1,
204            TokenKind::Rparen => 1,
205            TokenKind::Lbrace => 1,
206            TokenKind::Rbrace => 1,
207            TokenKind::Lbrack => 1,
208            TokenKind::Rbrack => 1,
209            TokenKind::Comment(ref s) => s.len() + 2,
210            TokenKind::BlockComment(ref s) => s.len() + 4,
211            TokenKind::PoundComment(ref s) => s.len() + 1,
212            TokenKind::IntConstant(ref s) => s.len(),
213            TokenKind::DoubleConstant(ref s) => s.len(),
214            TokenKind::NamespaceScope(ref s) => s.len(),
215            TokenKind::BaseType(ref s) => s.len(),
216            TokenKind::Literal(ref s) => s.len() + 2,
217            TokenKind::Identifier(ref s) => s.len(),
218            TokenKind::ListSeparator(_) => 1,
219            TokenKind::Invalid(_) => 1,
220            TokenKind::InvalidString(ref s) => s.len(),
221            TokenKind::Eof => 0,
222        }
223    }
224}
225
226impl TokenKind {
227    /// Creates a token from a string.
228    pub fn from_string(s: &str) -> Option<TokenKind> {
229        let tok = match s {
230            "include" => TokenKind::Include,
231            "cpp_include" => TokenKind::CppInclude,
232            "namespace" => TokenKind::Namespace,
233            "const" => TokenKind::Const,
234            "typedef" => TokenKind::Typedef,
235            "enum" => TokenKind::Enum,
236            "struct" => TokenKind::Struct,
237            "union" => TokenKind::Union,
238            "exception" => TokenKind::Exception,
239            "service" => TokenKind::Service,
240            "required" => TokenKind::Required,
241            "optional" => TokenKind::Optional,
242            "oneway" => TokenKind::Oneway,
243            "void" => TokenKind::Void,
244            "throws" => TokenKind::Throws,
245            "extends" => TokenKind::Extends,
246            "map" => TokenKind::Map,
247            "set" => TokenKind::Set,
248            "list" => TokenKind::List,
249            "cpp_type" => TokenKind::CppType,
250
251            // namespace scopes
252            "c_glib" => TokenKind::NamespaceScope(String::from("c_glib")),
253            "cpp" => TokenKind::NamespaceScope(String::from("cpp")),
254            "delphi" => TokenKind::NamespaceScope(String::from("delphi")),
255            "haxe" => TokenKind::NamespaceScope(String::from("haxe")),
256            "go" => TokenKind::NamespaceScope(String::from("go")),
257            "java" => TokenKind::NamespaceScope(String::from("java")),
258            "js" => TokenKind::NamespaceScope(String::from("js")),
259            "lua" => TokenKind::NamespaceScope(String::from("lua")),
260            "netstd" => TokenKind::NamespaceScope(String::from("netstd")),
261            "perl" => TokenKind::NamespaceScope(String::from("perl")),
262            "php" => TokenKind::NamespaceScope(String::from("php")),
263            "py" => TokenKind::NamespaceScope(String::from("py")),
264            "py.twisted" => TokenKind::NamespaceScope(String::from("py.twisted")),
265            "rb" => TokenKind::NamespaceScope(String::from("rb")),
266            "st" => TokenKind::NamespaceScope(String::from("st")),
267            "xsd" => TokenKind::NamespaceScope(String::from("xsd")),
268
269            // base types
270            "bool" => TokenKind::BaseType(String::from("bool")),
271            "byte" => TokenKind::BaseType(String::from("byte")),
272            "i8" => TokenKind::BaseType(String::from("i8")),
273            "i16" => TokenKind::BaseType(String::from("i16")),
274            "i32" => TokenKind::BaseType(String::from("i32")),
275            "i64" => TokenKind::BaseType(String::from("i64")),
276            "double" => TokenKind::BaseType(String::from("double")),
277            "string" => TokenKind::BaseType(String::from("string")),
278            "binary" => TokenKind::BaseType(String::from("binary")),
279            "uuid" => TokenKind::BaseType(String::from("uuid")),
280
281            _ => return None,
282        };
283
284        Some(tok)
285    }
286
287    /// Creates a token from a character.
288    pub fn from_char(c: char) -> Option<TokenKind> {
289        let tok = match c {
290            '=' => TokenKind::Assign,
291            ':' => TokenKind::Colon,
292            '<' => TokenKind::Less,
293            '>' => TokenKind::Greater,
294            ',' => TokenKind::ListSeparator(','),
295            ';' => TokenKind::ListSeparator(';'),
296            '(' => TokenKind::Lparen,
297            ')' => TokenKind::Rparen,
298            '{' => TokenKind::Lbrace,
299            '}' => TokenKind::Rbrace,
300            '[' => TokenKind::Lbrack,
301            ']' => TokenKind::Rbrack,
302            '*' => TokenKind::NamespaceScope(String::from("*")),
303            _ => return None,
304        };
305
306        Some(tok)
307    }
308}
thrift_analyzer/analyzer/token.rs

thrift_analyzer/analyzer/
token.rs
