rslint_lexer/
token.rs

1//! Token definitions for the lexer
2
3use crate::SyntaxKind;
4
5/// A single raw token such as `>>` or `||` or `"abc"`.
6#[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
7pub struct Token {
8    /// The kind of token this is.
9    pub kind: SyntaxKind,
10    /// How long the token is in bytes. For tokens with escape sequences
11    /// like strings with `\uXXXX` escapes, the length is the raw length, not considering the char backed by the escape.
12    pub len: usize,
13}
14
15impl Token {
16    /// Create a new token which has an exact length of 1.
17    pub fn single(kind: SyntaxKind) -> Self {
18        Self { kind, len: 1 }
19    }
20
21    /// Create a new token which has a specific length.
22    pub fn new(kind: SyntaxKind, len: usize) -> Self {
23        Self { kind, len }
24    }
25}
26
27macro_rules! tok {
28    ($tok:tt) => {
29        (Token::new(T![$tok], stringify!($tok).len()), None)
30    };
31    ($tok:ident, $len:expr) => {
32        (Token::new(SyntaxKind::$tok, $len), None)
33    };
34}
rslint_lexer/token.rs

rslint_lexer/
token.rs
