luau_lexer/token/
mod.rs

1//! The [`Token`] struct.
2
3mod r#impl;
4
5use lsp_types::Position;
6use smol_str::SmolStr;
7
8use crate::prelude::{Error, PositionExt};
9
10crate_reexport!(literal, keyword, symbol, operator, comment);
11
12/// A single token. Every [`lexable`](crate::lexer::Lexable) item becomes
13/// a token in [`Lexer::next_token()`](crate::lexer::Lexer::next_token).
14#[derive(Clone, Debug, Hash, PartialEq, Eq, PartialOrd, Ord)]
15#[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize))]
16pub struct Token {
17    /// The starting position of this token
18    pub start: Position,
19
20    /// The trivia before the token.
21    pub leading_trivia: Vec<Trivia>,
22
23    /// The actual info of the token.
24    pub token_type: TokenType,
25
26    /// The trivia after the token.
27    pub trailing_trivia: Vec<Trivia>,
28
29    /// The ending position of this token.
30    pub end: Position,
31}
32
33/// Trivia that can be before and after a token.
34#[derive(Clone, Debug, Hash, PartialEq, Eq, PartialOrd, Ord)]
35#[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize))]
36pub enum Trivia {
37    /// Spaces, be it whitespace, tabs, new lines, etc.
38    Spaces(SmolStr),
39
40    /// Comment, single or multi line.
41    Comment(Comment),
42}
43
44impl Token {
45    /// Creates an empty token with the specified type. This is only used when
46    /// creating tokens that don't have actual positions.
47    #[inline]
48    pub const fn empty(token_type: TokenType) -> Self {
49        Self {
50            start: Position::MAX,
51            leading_trivia: Vec::new(),
52            token_type,
53            trailing_trivia: Vec::new(),
54            end: Position::MAX,
55        }
56    }
57}
58
59impl PartialEq<TokenType> for Token {
60    fn eq(&self, other: &TokenType) -> bool {
61        &self.token_type == other
62    }
63}
64
65/// All token types.
66#[derive(Clone, Debug, Hash, PartialEq, Eq, PartialOrd, Ord)]
67#[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize))]
68#[allow(clippy::module_name_repetitions)] // `Type` is too general.
69pub enum TokenType {
70    /// An unknown type.
71    Error(Error),
72
73    /// A luau literal
74    Literal(Literal),
75
76    /// An identifier, like a variable name.
77    Identifier(SmolStr),
78
79    /// A comment
80    Comment(Comment),
81
82    /// A luau-reserved-keyword
83    Keyword(Keyword),
84
85    /// A word that can both be a keyword and an identifier.
86    PartialKeyword(PartialKeyword),
87
88    /// Symbols like `(` and `)`.
89    Symbol(Symbol),
90
91    /// Operators like `+` and `and`
92    Operator(Operator),
93
94    /// Compound operators like `+=` and `//=`
95    CompoundOperator(CompoundOperator),
96
97    /// The end of file token.
98    EndOfFile,
99}
100
101impl TokenType {
102    /// Turn this token type into a [`Token`] with the passed properties.
103    pub const fn into_token(
104        self,
105        start: Position,
106        end: Position,
107        leading_trivia: Vec<Trivia>,
108        trailing_trivia: Vec<Trivia>,
109    ) -> Token {
110        Token {
111            start,
112            leading_trivia,
113            token_type: self,
114            trailing_trivia,
115            end,
116        }
117    }
118}
119
120impl TokenType {
121    /// Try converting this token type into a string.
122    pub fn try_as_string(&self) -> Option<String> {
123        match self {
124            Self::Literal(literal) => match literal {
125                Literal::Number(luau_number) => match luau_number {
126                    LuauNumber::Plain(smol_str)
127                    | LuauNumber::Binary(smol_str)
128                    | LuauNumber::Hex(smol_str) => Some(smol_str.to_string()),
129                },
130                Literal::String(luau_string) => match luau_string {
131                    LuauString::SingleQuotes(smol_str)
132                    | LuauString::DoubleQuotes(smol_str)
133                    | LuauString::Backticks(smol_str)
134                    | LuauString::MultiLine(smol_str) => Some(smol_str.to_string()),
135                },
136                Literal::Boolean(true) => Some("true".to_string()),
137                Literal::Boolean(false) => Some("false".to_string()),
138            },
139            Self::Identifier(smol_str) => Some(smol_str.to_string()),
140            Self::Comment(comment) => match comment {
141                Comment::MultiLine(smol_str) | Comment::SingleLine(smol_str) => {
142                    Some(smol_str.to_string())
143                }
144            },
145            Self::Keyword(keyword) => Some(keyword.to_string()),
146            Self::PartialKeyword(partial_keyword) => Some(partial_keyword.to_string()),
147            Self::Symbol(symbol) => Some(symbol.to_string()),
148            Self::Operator(operator) => Some(operator.to_string()),
149            Self::CompoundOperator(compound_operator) => Some(compound_operator.to_string()),
150            Self::Error(_) | Self::EndOfFile => None,
151        }
152    }
153}
154
155impl_from!(TokenType <= {
156    Error(Error),
157    Literal(Literal),
158    Keyword(Keyword),
159    PartialKeyword(PartialKeyword),
160    Symbol(Symbol),
161    Operator(Operator),
162    CompoundOperator(CompoundOperator),
163});
luau_lexer/token/mod.rs

luau_lexer/token/
mod.rs
