strem/compiler/lexer/
token.rs

1//! Lexical unit information.
2//!
3
4#[derive(Clone, Debug, PartialEq)]
5pub enum TokenKind {
6    LeftParen,
7    RightParen,
8    LeftBrace,
9    RightBrace,
10    LeftBracket,
11    RightBracket,
12    LeftChevron,
13    RightChevron,
14    Comma,
15    Colon,
16    Star,
17    Percent,
18    Not,
19    And,
20    Or,
21    EndOfFile,
22    Integer,
23    Real,
24    Identifier,
25    NonEmpty,
26}
27
28/// Locational information used in a [`Token`].
29///
30/// This includes the row and the column number where the token begins.
31/// **Note**: The beginning of the source is located at (1, 0).
32#[derive(Clone, Debug, PartialEq)]
33pub struct Position(pub usize, pub usize);
34
35/// A lexical unit produced during tokenization by the lexical analyzer.
36#[derive(Clone, Debug, PartialEq)]
37pub struct Token {
38    pub kind: TokenKind,
39    pub position: Position,
40    pub lexeme: String,
41}
42
43impl Token {
44    pub fn new(kind: TokenKind, position: Position, lexeme: String) -> Self {
45        Token {
46            kind,
47            position,
48            lexeme,
49        }
50    }
51
52    pub fn eof(position: Position) -> Self {
53        Token {
54            kind: TokenKind::EndOfFile,
55            position,
56            lexeme: String::new(),
57        }
58    }
59}
strem/compiler/lexer/token.rs

strem/compiler/lexer/
token.rs
