goscript_parser/
token.rs

1#![allow(non_camel_case_types)]
2use std::fmt;
3
4pub const LOWEST_PREC: usize = 0; // non-operators
5pub const UNARY_PREC: usize = 6;
6pub const HIGHEST_PREC: usize = 7;
7
8#[derive(Hash, Eq, PartialEq, Clone)]
9pub enum Token {
10	// Special tokens
11	NONE,
12	ILLEGAL(TokenData),
13	EOF,
14	COMMENT(TokenData),
15
16	// Identifiers and basic type literals
17	IDENT(TokenData),  // main
18	INT(TokenData),    // 12345
19	FLOAT(TokenData),  // 123.45
20	IMAG(TokenData),   // 123.45i
21	CHAR(TokenData),   // 'a'
22	STRING(TokenData), // "abc"
23	// Operator
24	ADD, // +
25	SUB, // -
26	MUL, // *
27	QUO, // /
28	REM, // %
29
30	AND,     // &
31	OR,      // |
32	XOR,     // ^
33	SHL,     // <<
34	SHR,     // >>
35	AND_NOT, // &^
36
37	ADD_ASSIGN, // +=
38	SUB_ASSIGN, // -=
39	MUL_ASSIGN, // *=
40	QUO_ASSIGN, // /=
41	REM_ASSIGN, // %=
42
43	AND_ASSIGN,     // &=
44	OR_ASSIGN,      // |=
45	XOR_ASSIGN,     // ^=
46	SHL_ASSIGN,     // <<=
47	SHR_ASSIGN,     // >>=
48	AND_NOT_ASSIGN, // &^=
49
50	LAND,  // &&
51	LOR,   // ||
52	ARROW, // <-
53	INC,   // ++
54	DEC,   // --
55
56	EQL,    // ==
57	LSS,    // <
58	GTR,    // >
59	ASSIGN, // =
60	NOT,    // !
61
62	NEQ,      // !=
63	LEQ,      // <=
64	GEQ,      // >=
65	DEFINE,   // :=
66	ELLIPSIS, // ...
67
68	LPAREN, // (
69	LBRACK, // [
70	LBRACE, // {
71	COMMA,  // ,
72	PERIOD, // .
73
74	RPAREN,               // )
75	RBRACK,               // ]
76	RBRACE,               // }
77	SEMICOLON(TokenData), // ; true if SEMICOLON is NOT inserted by scanner
78	COLON,                // :
79
80	// Keywords
81	BREAK,
82	CASE,
83	CHAN,
84	CONST,
85	CONTINUE,
86
87	DEFAULT,
88	DEFER,
89	ELSE,
90	FALLTHROUGH,
91	FOR,
92
93	FUNC,
94	GO,
95	GOTO,
96	IF,
97	IMPORT,
98
99	INTERFACE,
100	MAP,
101	PACKAGE,
102	RANGE,
103	RETURN,
104
105	SELECT,
106	STRUCT,
107	SWITCH,
108	TYPE,
109	VAR,
110}
111
112pub enum TokenType {
113	Literal,
114	Operator,
115	Keyword,
116	Other,
117}
118
119impl Token {
120	pub fn token_property(&self) -> (TokenType, &str) {
121		match self {
122			Token::NONE => (TokenType::Other, "NONE"),
123			Token::ILLEGAL(_) => (TokenType::Other, "ILLEGAL"),
124			Token::EOF => (TokenType::Other, "EOF"),
125			Token::COMMENT(_) => (TokenType::Other, "COMMENT"),
126			Token::IDENT(_) => (TokenType::Literal, "IDENT"),
127			Token::INT(_) => (TokenType::Literal, "INT"),
128			Token::FLOAT(_) => (TokenType::Literal, "FLOAT"),
129			Token::IMAG(_) => (TokenType::Literal, "IMAG"),
130			Token::CHAR(_) => (TokenType::Literal, "CHAR"),
131			Token::STRING(_) => (TokenType::Literal, "STRING"),
132			Token::ADD => (TokenType::Operator, "+"),
133			Token::SUB => (TokenType::Operator, "-"),
134			Token::MUL => (TokenType::Operator, "*"),
135			Token::QUO => (TokenType::Operator, "/"),
136			Token::REM => (TokenType::Operator, "%"),
137			Token::AND => (TokenType::Operator, "&"),
138			Token::OR => (TokenType::Operator, "|"),
139			Token::XOR => (TokenType::Operator, "^"),
140			Token::SHL => (TokenType::Operator, "<<"),
141			Token::SHR => (TokenType::Operator, ">>"),
142			Token::AND_NOT => (TokenType::Operator, "&^"),
143			Token::ADD_ASSIGN => (TokenType::Operator, "+="),
144			Token::SUB_ASSIGN => (TokenType::Operator, "-="),
145			Token::MUL_ASSIGN => (TokenType::Operator, "*="),
146			Token::QUO_ASSIGN => (TokenType::Operator, "/="),
147			Token::REM_ASSIGN => (TokenType::Operator, "%="),
148			Token::AND_ASSIGN => (TokenType::Operator, "&="),
149			Token::OR_ASSIGN => (TokenType::Operator, "|="),
150			Token::XOR_ASSIGN => (TokenType::Operator, "^="),
151			Token::SHL_ASSIGN => (TokenType::Operator, "<<="),
152			Token::SHR_ASSIGN => (TokenType::Operator, ">>="),
153			Token::AND_NOT_ASSIGN => (TokenType::Operator, "&^="),
154			Token::LAND => (TokenType::Operator, "&&"),
155			Token::LOR => (TokenType::Operator, "||"),
156			Token::ARROW => (TokenType::Operator, "<-"),
157			Token::INC => (TokenType::Operator, "++"),
158			Token::DEC => (TokenType::Operator, "--"),
159			Token::EQL => (TokenType::Operator, "=="),
160			Token::LSS => (TokenType::Operator, "<"),
161			Token::GTR => (TokenType::Operator, ">"),
162			Token::ASSIGN => (TokenType::Operator, "="),
163			Token::NOT => (TokenType::Operator, "!"),
164			Token::NEQ => (TokenType::Operator, "!="),
165			Token::LEQ => (TokenType::Operator, "<="),
166			Token::GEQ => (TokenType::Operator, ">="),
167			Token::DEFINE => (TokenType::Operator, ":="),
168			Token::ELLIPSIS => (TokenType::Operator, "..."),
169			Token::LPAREN => (TokenType::Operator, "("),
170			Token::LBRACK => (TokenType::Operator, "["),
171			Token::LBRACE => (TokenType::Operator, "{"),
172			Token::COMMA => (TokenType::Operator, ","),
173			Token::PERIOD => (TokenType::Operator, "."),
174			Token::RPAREN => (TokenType::Operator, ")"),
175			Token::RBRACK => (TokenType::Operator, "]"),
176			Token::RBRACE => (TokenType::Operator, "}"),
177			Token::SEMICOLON(_) => (TokenType::Operator, ";"),
178			Token::COLON => (TokenType::Operator, ":"),
179			Token::BREAK => (TokenType::Keyword, "break"),
180			Token::CASE => (TokenType::Keyword, "case"),
181			Token::CHAN => (TokenType::Keyword, "chan"),
182			Token::CONST => (TokenType::Keyword, "const"),
183			Token::CONTINUE => (TokenType::Keyword, "continue"),
184			Token::DEFAULT => (TokenType::Keyword, "default"),
185			Token::DEFER => (TokenType::Keyword, "defer"),
186			Token::ELSE => (TokenType::Keyword, "else"),
187			Token::FALLTHROUGH => (TokenType::Keyword, "fallthrough"),
188			Token::FOR => (TokenType::Keyword, "for"),
189			Token::FUNC => (TokenType::Keyword, "func"),
190			Token::GO => (TokenType::Keyword, "go"),
191			Token::GOTO => (TokenType::Keyword, "goto"),
192			Token::IF => (TokenType::Keyword, "if"),
193			Token::IMPORT => (TokenType::Keyword, "import"),
194			Token::INTERFACE => (TokenType::Keyword, "interface"),
195			Token::MAP => (TokenType::Keyword, "map"),
196			Token::PACKAGE => (TokenType::Keyword, "package"),
197			Token::RANGE => (TokenType::Keyword, "range"),
198			Token::RETURN => (TokenType::Keyword, "return"),
199			Token::SELECT => (TokenType::Keyword, "select"),
200			Token::STRUCT => (TokenType::Keyword, "struct"),
201			Token::SWITCH => (TokenType::Keyword, "switch"),
202			Token::TYPE => (TokenType::Keyword, "type"),
203			Token::VAR => (TokenType::Keyword, "var"),
204		}
205	}
206
207	pub fn ident_token(ident: String) -> Token {
208		match ident.as_str() {
209			"break" => Token::BREAK,
210			"case" => Token::CASE,
211			"chan" => Token::CHAN,
212			"const" => Token::CONST,
213			"continue" => Token::CONTINUE,
214			"default" => Token::DEFAULT,
215			"defer" => Token::DEFER,
216			"else" => Token::ELSE,
217			"fallthrough" => Token::FALLTHROUGH,
218			"for" => Token::FOR,
219			"func" => Token::FUNC,
220			"go" => Token::GO,
221			"goto" => Token::GOTO,
222			"if" => Token::IF,
223			"import" => Token::IMPORT,
224			"interface" => Token::INTERFACE,
225			"map" => Token::MAP,
226			"package" => Token::PACKAGE,
227			"range" => Token::RANGE,
228			"return" => Token::RETURN,
229			"select" => Token::SELECT,
230			"struct" => Token::STRUCT,
231			"switch" => Token::SWITCH,
232			"type" => Token::TYPE,
233			"var" => Token::VAR,
234			_ => Token::IDENT(ident.into()),
235		}
236	}
237
238	pub fn int1() -> Token {
239		Token::INT("1".to_string().into())
240	}
241
242	pub fn precedence(&self) -> usize {
243		match self {
244			Token::LOR => 1,
245			Token::LAND => 2,
246			Token::EQL | Token::NEQ | Token::LSS | Token::LEQ | Token::GTR | Token::GEQ => 3,
247			Token::ADD | Token::SUB | Token::OR | Token::XOR => 4,
248			Token::MUL
249			| Token::QUO
250			| Token::REM
251			| Token::SHL
252			| Token::SHR
253			| Token::AND
254			| Token::AND_NOT => 5,
255			_ => LOWEST_PREC,
256		}
257	}
258
259	pub fn text(&self) -> &str {
260		let (_, t) = self.token_property();
261		t
262	}
263
264	pub fn is_literal(&self) -> bool {
265		match self.token_property().0 {
266			TokenType::Literal => true,
267			_ => false,
268		}
269	}
270
271	pub fn is_operator(&self) -> bool {
272		match self.token_property().0 {
273			TokenType::Operator => true,
274			_ => false,
275		}
276	}
277
278	pub fn is_keyword(&self) -> bool {
279		match self.token_property().0 {
280			TokenType::Keyword => true,
281			_ => false,
282		}
283	}
284
285	pub fn get_literal(&self) -> &str {
286		match self {
287			Token::INT(l) => l.as_str(),
288			Token::FLOAT(l) => l.as_str(),
289			Token::IMAG(l) => l.as_str(),
290			Token::CHAR(l) => l.as_str(),
291			Token::STRING(l) => l.as_str(),
292			_ => "",
293		}
294	}
295
296	pub fn is_stmt_start(&self) -> bool {
297		match self {
298			Token::BREAK => true,
299			Token::CONST => true,
300			Token::CONTINUE => true,
301			Token::DEFER => true,
302			Token::FALLTHROUGH => true,
303			Token::FOR => true,
304			Token::GO => true,
305			Token::GOTO => true,
306			Token::IF => true,
307			Token::RETURN => true,
308			Token::SELECT => true,
309			Token::SWITCH => true,
310			Token::TYPE => true,
311			Token::VAR => true,
312			_ => false,
313		}
314	}
315
316	pub fn is_decl_start(&self) -> bool {
317		match self {
318			Token::CONST => true,
319			Token::TYPE => true,
320			Token::VAR => true,
321			_ => false,
322		}
323	}
324
325	pub fn is_expr_end(&self) -> bool {
326		match self {
327			Token::COMMA => true,
328			Token::COLON => true,
329			Token::SEMICOLON(_) => true,
330			Token::RPAREN => true,
331			Token::RBRACK => true,
332			Token::RBRACE => true,
333			_ => false,
334		}
335	}
336}
337
338impl fmt::Display for Token {
339	fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
340		let text = self.text();
341		match self {
342			Token::IDENT(l)
343			| Token::INT(l)
344			| Token::FLOAT(l)
345			| Token::IMAG(l)
346			| Token::CHAR(l)
347			| Token::STRING(l) => f.write_str(l.as_str()),
348			_ => write!(f, "{}", text),
349		}
350	}
351}
352
353impl fmt::Debug for Token {
354	fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
355		let text = self.text();
356		match self {
357			Token::IDENT(l)
358			| Token::INT(l)
359			| Token::FLOAT(l)
360			| Token::IMAG(l)
361			| Token::CHAR(l)
362			| Token::STRING(l) => write!(f, "{} {}", text, l.as_str()),
363			Token::SEMICOLON(real) if !*real.as_bool() => write!(f, "\"{}(inserted)\"", text),
364			token if token.is_operator() || token.is_keyword() => write!(f, "\"{}\"", text),
365			_ => write!(f, "{}", text),
366		}
367	}
368}
369
370#[derive(Hash, Eq, PartialEq, Clone, Debug)]
371enum RawTokenData {
372	Bool(bool),
373	Str(String),
374	StrStr(String, String),
375	StrChar(String, char),
376}
377
378#[derive(Hash, Eq, PartialEq, Clone, Debug)]
379pub struct TokenData(Box<RawTokenData>);
380
381impl From<bool> for TokenData {
382	fn from(b: bool) -> Self {
383		TokenData(Box::new(RawTokenData::Bool(b)))
384	}
385}
386
387impl From<String> for TokenData {
388	fn from(s: String) -> Self {
389		TokenData(Box::new(RawTokenData::Str(s)))
390	}
391}
392
393impl From<(String, String)> for TokenData {
394	fn from(ss: (String, String)) -> Self {
395		TokenData(Box::new(RawTokenData::StrStr(ss.0, ss.1)))
396	}
397}
398
399impl From<(String, char)> for TokenData {
400	fn from(ss: (String, char)) -> Self {
401		TokenData(Box::new(RawTokenData::StrChar(ss.0, ss.1)))
402	}
403}
404
405impl AsRef<bool> for TokenData {
406	fn as_ref(&self) -> &bool {
407		self.as_bool()
408	}
409}
410
411impl AsRef<String> for TokenData {
412	fn as_ref(&self) -> &String {
413		self.as_str()
414	}
415}
416
417impl AsMut<String> for TokenData {
418	fn as_mut(&mut self) -> &mut String {
419		self.as_str_mut()
420	}
421}
422
423impl TokenData {
424	pub fn as_bool(&self) -> &bool {
425		match self.0.as_ref() {
426			RawTokenData::Bool(b) => b,
427			_ => unreachable!(),
428		}
429	}
430
431	pub fn as_str(&self) -> &String {
432		match self.0.as_ref() {
433			RawTokenData::Str(s) => s,
434			RawTokenData::StrStr(s, _) => s,
435			RawTokenData::StrChar(s, _) => s,
436			_ => unreachable!(),
437		}
438	}
439
440	pub fn as_str_mut(&mut self) -> &mut String {
441		match self.0.as_mut() {
442			RawTokenData::Str(s) => s,
443			RawTokenData::StrStr(s, _) => s,
444			RawTokenData::StrChar(s, _) => s,
445			_ => unreachable!(),
446		}
447	}
448
449	pub fn as_str_str(&self) -> (&String, &String) {
450		match self.0.as_ref() {
451			RawTokenData::StrStr(s1, s2) => (s1, s2),
452			_ => unreachable!(),
453		}
454	}
455
456	pub fn as_str_char(&self) -> (&String, &char) {
457		match self.0.as_ref() {
458			RawTokenData::StrChar(s, c) => (s, c),
459			_ => unreachable!(),
460		}
461	}
462}
463
464#[cfg(test)]
465mod test {
466	use super::*;
467
468	#[test]
469	fn token_test() {
470		print!(
471			"testxxxxx \n{}\n{}\n{}\n{}\n. ",
472			Token::ILLEGAL("asd".to_string().into()),
473			Token::SWITCH,
474			Token::IDENT("some_var".to_string().into()),
475			Token::FLOAT("3.14".to_string().into()),
476		);
477	}
478}
goscript_parser/token.rs

goscript_parser/
token.rs
