rush_parser/
token.rs

1use std::fmt::{self, Debug, Display, Formatter};
2
3use crate::Span;
4
5#[derive(Clone, Copy, PartialEq)]
6pub struct Token<'src> {
7    pub kind: TokenKind<'src>,
8    pub span: Span<'src>,
9}
10
11impl<'src> Token<'src> {
12    pub fn new(kind: TokenKind<'src>, span: Span<'src>) -> Self {
13        Self { kind, span }
14    }
15
16    pub fn dummy() -> Self {
17        Self {
18            kind: TokenKind::default(),
19            span: Span::dummy(),
20        }
21    }
22}
23
24impl Debug for Token<'_> {
25    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
26        write!(
27            f,
28            "{:<15} (l:{}:{} -- l:{}:{})",
29            format!("{:?}", self.kind),
30            self.span.start.line,
31            self.span.start.column,
32            self.span.end.line,
33            self.span.end.column
34        )
35    }
36}
37
38#[derive(Debug, Clone, Copy, PartialEq, Default)]
39pub enum TokenKind<'src> {
40    /// End of file
41    #[default]
42    Eof,
43
44    /// An identifier
45    Ident(&'src str),
46    /// An int literal
47    Int(i64),
48    /// A float literal
49    Float(f64),
50    /// A char literal
51    Char(u8),
52
53    /// `true`
54    True,
55    /// `false`
56    False,
57    /// `fn`
58    Fn,
59    /// `let`
60    Let,
61    /// `mut`
62    Mut,
63    /// `return`
64    Return,
65    /// `loop`
66    Loop,
67    /// `while`
68    While,
69    /// `for`
70    For,
71    /// `break`
72    Break,
73    /// `continue`
74    Continue,
75    /// `if`
76    If,
77    /// `else`
78    Else,
79    /// `as`
80    As,
81
82    /// (
83    LParen,
84    /// )
85    RParen,
86    /// {
87    LBrace,
88    /// }
89    RBrace,
90
91    /// ->
92    Arrow,
93    /// ,
94    Comma,
95    /// :
96    Colon,
97    /// ;
98    Semicolon,
99
100    /// !
101    Not,
102    /// -
103    Minus,
104    /// +
105    Plus,
106    /// *
107    Star,
108    /// /
109    Slash,
110    /// %
111    Percent,
112    /// **
113    Pow,
114    /// ==
115    Eq,
116    /// !=
117    Neq,
118    /// <
119    Lt,
120    /// >
121    Gt,
122    /// <=
123    Lte,
124    /// >=
125    Gte,
126    /// <<
127    Shl,
128    /// >>
129    Shr,
130    /// |
131    BitOr,
132    /// &
133    BitAnd,
134    /// ^
135    BitXor,
136    /// &&
137    And,
138    /// ||
139    Or,
140
141    /// =
142    Assign,
143    /// +=
144    PlusAssign,
145    /// -=
146    MinusAssign,
147    /// *=
148    MulAssign,
149    /// /=
150    DivAssign,
151    /// %=
152    RemAssign,
153    /// **=
154    PowAssign,
155    /// <<=
156    ShlAssign,
157    /// >>=
158    ShrAssign,
159    /// |=
160    BitOrAssign,
161    /// &=
162    BitAndAssign,
163    /// ^=
164    BitXorAssign,
165}
166
167impl<'src> TokenKind<'src> {
168    pub fn spanned(self, span: Span<'src>) -> Token<'src> {
169        Token { kind: self, span }
170    }
171
172    pub(crate) fn prec(&self) -> (u8, u8) {
173        match self {
174            TokenKind::Assign
175            | TokenKind::PlusAssign
176            | TokenKind::MinusAssign
177            | TokenKind::MulAssign
178            | TokenKind::DivAssign
179            | TokenKind::RemAssign
180            | TokenKind::PowAssign
181            | TokenKind::ShlAssign
182            | TokenKind::ShrAssign
183            | TokenKind::BitOrAssign
184            | TokenKind::BitAndAssign
185            | TokenKind::BitXorAssign => (1, 2),
186            TokenKind::Or => (3, 4),
187            TokenKind::And => (5, 6),
188            TokenKind::BitOr => (7, 8),
189            TokenKind::BitXor => (9, 10),
190            TokenKind::BitAnd => (11, 12),
191            TokenKind::Eq | TokenKind::Neq => (13, 14),
192            TokenKind::Lt | TokenKind::Gt | TokenKind::Lte | TokenKind::Gte => (15, 16),
193            TokenKind::Shl | TokenKind::Shr => (17, 18),
194            TokenKind::Plus | TokenKind::Minus => (19, 20),
195            TokenKind::Star | TokenKind::Slash | TokenKind::Percent => (21, 22),
196            TokenKind::As => (23, 24),
197            TokenKind::Pow => (26, 25), // inverse order for right associativity
198            TokenKind::LParen => (28, 29),
199            _ => (0, 0),
200        }
201    }
202}
203
204impl Display for TokenKind<'_> {
205    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
206        match self {
207            Self::Eof => write!(f, "EOF"),
208            Self::Ident(ident) => write!(f, "{ident}"),
209            Self::Int(num) => write!(f, "{num}"),
210            Self::Float(num) => write!(f, "{num}"),
211            Self::Char(b'\\') => write!(f, "'\\\\'"),
212            Self::Char(b'\x08') => write!(f, "'\\b'"),
213            Self::Char(b'\n') => write!(f, "'\\n'"),
214            Self::Char(b'\r') => write!(f, "'\\r'"),
215            Self::Char(b'\t') => write!(f, "'\\t'"),
216            Self::Char(b'\'') => write!(f, "'\\''"),
217            Self::Char(char @ b' '..=b'~') => write!(f, "'{}'", *char as char),
218            Self::Char(char) => write!(f, "'\\x{char:x}'"),
219            Self::True => write!(f, "true"),
220            Self::False => write!(f, "false"),
221            Self::Fn => write!(f, "fn"),
222            Self::Let => write!(f, "let"),
223            Self::Mut => write!(f, "mut"),
224            Self::Return => write!(f, "return"),
225            Self::Loop => write!(f, "loop"),
226            Self::While => write!(f, "while"),
227            Self::For => write!(f, "for"),
228            Self::Break => write!(f, "break"),
229            Self::Continue => write!(f, "continue"),
230            Self::If => write!(f, "if"),
231            Self::Else => write!(f, "else"),
232            Self::As => write!(f, "as"),
233            Self::LParen => write!(f, "("),
234            Self::RParen => write!(f, ")"),
235            Self::LBrace => write!(f, "{{"),
236            Self::RBrace => write!(f, "}}"),
237            Self::Arrow => write!(f, "->"),
238            Self::Comma => write!(f, ","),
239            Self::Colon => write!(f, ":"),
240            Self::Semicolon => write!(f, ";"),
241            Self::Not => write!(f, "!"),
242            Self::Minus => write!(f, "-"),
243            Self::Plus => write!(f, "+"),
244            Self::Star => write!(f, "*"),
245            Self::Slash => write!(f, "/"),
246            Self::Percent => write!(f, "%"),
247            Self::Pow => write!(f, "**"),
248            Self::Eq => write!(f, "=="),
249            Self::Neq => write!(f, "!="),
250            Self::Lt => write!(f, "<"),
251            Self::Gt => write!(f, ">"),
252            Self::Lte => write!(f, "<="),
253            Self::Gte => write!(f, ">="),
254            Self::Shl => write!(f, "<<"),
255            Self::Shr => write!(f, ">>"),
256            Self::BitOr => write!(f, "|"),
257            Self::BitAnd => write!(f, "&"),
258            Self::BitXor => write!(f, "^"),
259            Self::And => write!(f, "&&"),
260            Self::Or => write!(f, "||"),
261            Self::Assign => write!(f, "="),
262            Self::PlusAssign => write!(f, "+="),
263            Self::MinusAssign => write!(f, "-="),
264            Self::MulAssign => write!(f, "*="),
265            Self::DivAssign => write!(f, "/="),
266            Self::RemAssign => write!(f, "%="),
267            Self::PowAssign => write!(f, "**="),
268            Self::ShlAssign => write!(f, "<<="),
269            Self::ShrAssign => write!(f, ">>="),
270            Self::BitOrAssign => write!(f, "|="),
271            Self::BitAndAssign => write!(f, "&="),
272            Self::BitXorAssign => write!(f, "^="),
273        }
274    }
275}
276
277#[cfg(test)]
278mod tests {
279    use super::*;
280
281    #[test]
282    fn char_display() {
283        assert_eq!(TokenKind::Char(b'\\').to_string(), r"'\\'");
284        assert_eq!(TokenKind::Char(0x08).to_string(), r"'\b'");
285        assert_eq!(TokenKind::Char(b'\n').to_string(), r"'\n'");
286        assert_eq!(TokenKind::Char(b'\r').to_string(), r"'\r'");
287        assert_eq!(TokenKind::Char(b'\t').to_string(), r"'\t'");
288        assert_eq!(TokenKind::Char(b'\'').to_string(), r"'\''");
289        assert_eq!(TokenKind::Char(b'a').to_string(), "'a'");
290        assert_eq!(TokenKind::Char(b'0').to_string(), "'0'");
291        assert_eq!(TokenKind::Char(b' ').to_string(), "' '");
292        assert_eq!(TokenKind::Char(b'~').to_string(), "'~'");
293        assert_eq!(TokenKind::Char(0x7f).to_string(), r"'\x7f'");
294        assert_eq!(TokenKind::Char(0x1b).to_string(), r"'\x1b'");
295    }
296}
rush_parser/token.rs

rush_parser/
token.rs
