lang_interpreter/lexer/
token.rs

1use crate::lexer::CodePosition;
2use std::fmt::{Display, Formatter};
3
4#[derive(Debug, Copy, Clone, Eq, PartialEq, Hash)]
5pub enum TokenType {
6    Other,
7
8    LiteralNull,
9    LiteralText,
10    LiteralNumber,
11
12    ArgumentSeparator,
13
14    EscapeSequence,
15
16    ParserFunctionIdentifier,
17
18    Identifier,
19
20    Operator,
21
22    Assignment,
23
24    OpeningBracket,
25    ClosingBracket,
26
27    OpeningBlockBracket,
28    ClosingBlockBracket,
29
30    SingleLineTextQuotes,
31
32    StartMultilineText,
33    EndMultilineText,
34
35    StartComment,
36    StartDocComment,
37    EndComment,
38
39    LineContinuation,
40
41    Whitespace,
42    Eol,
43    Eof,
44
45    LexerError,
46}
47
48impl Display for TokenType {
49    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
50        f.write_str(match self {
51            TokenType::Other => "OTHER",
52
53            TokenType::LiteralNull => "LITERAL_NULL",
54            TokenType::LiteralText => "LITERAL_TEXT",
55            TokenType::LiteralNumber => "LITERAL_NUMBER",
56
57            TokenType::ArgumentSeparator => "ARGUMENT_SEPARATOR",
58
59            TokenType::EscapeSequence => "ESCAPE_SEQUENCE",
60
61            TokenType::ParserFunctionIdentifier => "PARSER_FUNCTION_IDENTIFIER",
62
63            TokenType::Identifier => "IDENTIFIER",
64
65            TokenType::Operator => "OPERATOR",
66
67            TokenType::Assignment => "ASSIGNMENT",
68
69            TokenType::OpeningBracket => "OPENING_BRACKET",
70            TokenType::ClosingBracket => "CLOSING_BRACKET",
71
72            TokenType::OpeningBlockBracket => "OPENING_BLOCK_BRACKET",
73            TokenType::ClosingBlockBracket => "CLOSING_BLOCK_BRACKET",
74
75            TokenType::SingleLineTextQuotes => "SINGLE_LINE_TEXT_QUOTES",
76
77            TokenType::StartMultilineText => "START_MULTILINE_TEXT",
78            TokenType::EndMultilineText => "END_MULTILINE_TEXT",
79
80            TokenType::StartComment => "START_COMMENT",
81            TokenType::StartDocComment => "START_DOC_COMMENT",
82            TokenType::EndComment => "END_COMMENT",
83
84            TokenType::LineContinuation => "LINE_CONTINUATION",
85
86            TokenType::Whitespace => "WHITESPACE",
87            TokenType::Eol => "EOL",
88            TokenType::Eof => "EOF",
89
90            TokenType::LexerError => "LEXER_ERROR",
91        })
92    }
93}
94
95#[derive(Debug, Clone, Eq, PartialEq, Hash)]
96pub struct Token {
97    pos: CodePosition,
98    value: Box<str>,
99    token_type: TokenType,
100}
101
102impl Token {
103    pub fn new(pos: CodePosition, value: &str, token_type: TokenType) -> Self {
104        Self {
105            pos,
106            value: value.into(),
107            token_type,
108        }
109    }
110
111    pub fn to_raw_string(&self) -> &str {
112        match self.token_type {
113            TokenType::LexerError => "",
114            _ => &self.value,
115        }
116    }
117
118    pub fn pos(&self) -> CodePosition {
119        self.pos
120    }
121
122    pub fn value(&self) -> &str {
123        &self.value
124    }
125
126    pub fn token_type(&self) -> TokenType {
127        self.token_type
128    }
129}
130
131impl Display for Token {
132    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
133        write!(
134            f, "Token ({:>30} at {}): \"{}\"",
135            format!("{}", self.token_type), self.pos, self.value
136        )
137    }
138}
lang_interpreter/lexer/token.rs

lang_interpreter/lexer/
token.rs
