mini_builder_rs/tokenizer/
token.rs

1#[derive(Debug, Clone, Copy, PartialEq, Eq)]
2pub enum TokenType {
3	/// anything that is not a directive
4	Source,
5	/// reserved symbols - characters
6	QuestionMark,
7	Colon,
8	Comma,
9	Dot,
10	LBracket,
11	RBracket,
12	SquareLBracket,
13	SquareRBracket,
14	/// reserved symbols - arithmetics
15	Plus,
16	Minus,
17	Star,
18	Slash,
19	/// reserved symbols - logical operators
20	Or,
21	And,
22	Not,
23	/// reserved symbols - assignments
24	Assign,
25	/// reserved symbols - directive definers
26	Open,
27	Close,
28	Pound,
29	At,
30	/// reserved symbols - comparisons
31	Equals,
32	NotEquals,
33	GreaterThan,
34	SmallerThan,
35	GreaterEquals,
36	SmallerEquals,
37	/// reserved words
38	If,
39	Else,
40	Elif,
41	For,
42	/// identifiers and literals
43	Identifier,
44	FilePath,
45	StringLiteral,
46	NumericalLiteral,
47	NoneLiteral,
48}
49
50impl TokenType {
51	pub fn is_comparison_token(&self) -> bool {
52		match *self {
53			Self::Equals
54			| Self::NotEquals
55			| Self::GreaterThan
56			| Self::SmallerThan
57			| Self::GreaterEquals
58			| Self::SmallerEquals => true,
59			_ => false,
60		}
61	}
62}
63
64pub struct Token<'a> {
65	pub tt: TokenType,
66	pub content: &'a str,
67	pub location: usize,
68}
69
70impl<'a> Token<'a> {
71	pub fn new(tt: TokenType, content: &'a str, location: usize) -> Self {
72		Self {
73			tt,
74			content: content,
75			location,
76		}
77	}
78}
mini_builder_rs/tokenizer/token.rs

mini_builder_rs/tokenizer/
token.rs
