mq_lang/lexer/
token.rs

1use std::fmt::{self, Display, Formatter};
2
3use itertools::Itertools;
4use smol_str::SmolStr;
5
6#[cfg(feature = "ast-json")]
7use crate::ArenaId;
8use crate::{module::ModuleId, number::Number, range::Range};
9#[cfg(feature = "ast-json")]
10use serde::{Deserialize, Serialize};
11
12#[cfg_attr(feature = "ast-json", derive(Serialize, Deserialize))]
13#[derive(Debug, Clone, PartialOrd, PartialEq, Ord, Eq)]
14pub enum StringSegment {
15    Text(String, Range),
16    Expr(SmolStr, Range),
17}
18
19impl Display for StringSegment {
20    fn fmt(&self, f: &mut Formatter<'_>) -> Result<(), fmt::Error> {
21        match self {
22            StringSegment::Text(text, _) => write!(f, "{}", text),
23            StringSegment::Expr(expr, _) => write!(f, "${{{}}}", expr),
24        }
25    }
26}
27
28#[cfg(feature = "ast-json")]
29fn default_module_id() -> ModuleId {
30    ArenaId::new(0)
31}
32
33#[cfg_attr(feature = "ast-json", derive(Serialize, Deserialize))]
34#[derive(PartialEq, Eq, PartialOrd, Ord, Debug, Clone)]
35pub struct Token {
36    pub range: Range,
37    pub kind: TokenKind,
38    #[cfg_attr(
39        feature = "ast-json",
40        serde(skip_serializing, skip_deserializing, default = "default_module_id")
41    )]
42    pub module_id: ModuleId,
43}
44
45#[cfg_attr(feature = "ast-json", derive(Serialize, Deserialize))]
46#[derive(PartialEq, Eq, PartialOrd, Ord, Debug, Clone)]
47/// Represents the kind of a token in the mq language.
48///
49/// TokenKind variants are sorted alphabetically for maintainability.
50pub enum TokenKind {
51    And,
52    Asterisk,
53    BoolLiteral(bool),
54    Break,
55    Catch,
56    Coalesce,
57    Colon,
58    DoubleColon,
59    DoubleSlashEqual,
60    Comma,
61    Comment(String),
62    Continue,
63    Def,
64    Do,
65    Elif,
66    Else,
67    End,
68    Env(SmolStr),
69    Eof,
70    Equal,
71    EqEq,
72    Fn,
73    Foreach,
74    Gt,
75    Gte,
76    Ident(SmolStr),
77    If,
78    Include,
79    InterpolatedString(Vec<StringSegment>),
80    Import,
81    LBrace,
82    LBracket,
83    Let,
84    Loop,
85    Lt,
86    Lte,
87    Macro,
88    Match,
89    Module,
90    Minus,
91    MinusEqual,
92    NeEq,
93    NewLine,
94    Nodes,
95    None,
96    Not,
97    NumberLiteral(Number),
98    Or,
99    Percent,
100    PercentEqual,
101    Pipe,
102    PipeEqual,
103    Plus,
104    PlusEqual,
105    Question,
106    Quote,
107    RBrace,
108    RangeOp,
109    RBracket,
110    RParen,
111    Selector(SmolStr),
112    Self_,
113    SemiColon,
114    Slash,
115    SlashEqual,
116    StringLiteral(String),
117    StarEqual,
118    Tab(usize),
119    Try,
120    Unquote,
121    Whitespace(usize),
122    While,
123    LParen,
124    Var,
125}
126
127impl Token {
128    #[inline(always)]
129    pub fn is_eof(&self) -> bool {
130        matches!(self.kind, TokenKind::Eof)
131    }
132
133    #[inline(always)]
134    pub fn is_selector(&self) -> bool {
135        matches!(self.kind, TokenKind::Selector(_))
136    }
137}
138
139impl Display for Token {
140    fn fmt(&self, f: &mut Formatter<'_>) -> Result<(), fmt::Error> {
141        write!(f, "{}", self.kind)
142    }
143}
144
145impl Display for TokenKind {
146    fn fmt(&self, f: &mut Formatter<'_>) -> Result<(), fmt::Error> {
147        match &self {
148            TokenKind::And => write!(f, "&&"),
149            TokenKind::Or => write!(f, "||"),
150            TokenKind::Not => write!(f, "!"),
151            TokenKind::Asterisk => write!(f, "*"),
152            TokenKind::BoolLiteral(b) => write!(f, "{}", b),
153            TokenKind::Break => write!(f, "break"),
154            TokenKind::Colon => write!(f, ":"),
155            TokenKind::Comma => write!(f, ","),
156            TokenKind::Continue => write!(f, "continue"),
157            TokenKind::Coalesce => write!(f, "??"),
158            TokenKind::Comment(comment) => write!(f, "# {}", comment.trim()),
159            TokenKind::Def => write!(f, "def"),
160            TokenKind::Do => write!(f, "do"),
161            TokenKind::DoubleColon => write!(f, "::"),
162            TokenKind::DoubleSlashEqual => write!(f, "//="),
163            TokenKind::Elif => write!(f, "elif"),
164            TokenKind::Else => write!(f, "else"),
165            TokenKind::End => write!(f, "end"),
166            TokenKind::Env(env) => write!(f, "${}", env),
167            TokenKind::Eof => write!(f, ""),
168            TokenKind::Equal => write!(f, "="),
169            TokenKind::EqEq => write!(f, "=="),
170            TokenKind::Fn => write!(f, "fn"),
171            TokenKind::Foreach => write!(f, "foreach"),
172            TokenKind::Ident(ident) => write!(f, "{}", ident),
173            TokenKind::If => write!(f, "if"),
174            TokenKind::Include => write!(f, "include"),
175            TokenKind::Import => write!(f, "import"),
176            TokenKind::InterpolatedString(segments) => {
177                write!(f, "{}", segments.iter().join(""))
178            }
179            TokenKind::Lt => write!(f, "<"),
180            TokenKind::Lte => write!(f, "<="),
181            TokenKind::Gt => write!(f, ">"),
182            TokenKind::Gte => write!(f, ">="),
183            TokenKind::LBracket => write!(f, "["),
184            TokenKind::LParen => write!(f, "("),
185            TokenKind::Let => write!(f, "let"),
186            TokenKind::Loop => write!(f, "loop"),
187            TokenKind::Macro => write!(f, "macro"),
188            TokenKind::Match => write!(f, "match"),
189            TokenKind::Module => write!(f, "module"),
190            TokenKind::Minus => write!(f, "-"),
191            TokenKind::MinusEqual => write!(f, "-="),
192            TokenKind::Slash => write!(f, "/"),
193            TokenKind::SlashEqual => write!(f, "/="),
194            TokenKind::Percent => write!(f, "%"),
195            TokenKind::PercentEqual => write!(f, "%="),
196            TokenKind::NeEq => write!(f, "!="),
197            TokenKind::NewLine => writeln!(f),
198            TokenKind::Nodes => write!(f, "nodes"),
199            TokenKind::None => write!(f, "None"),
200            TokenKind::NumberLiteral(n) => write!(f, "{}", n),
201            TokenKind::Plus => write!(f, "+"),
202            TokenKind::PlusEqual => write!(f, "+="),
203            TokenKind::Pipe => write!(f, "|"),
204            TokenKind::PipeEqual => write!(f, "|="),
205            TokenKind::Quote => write!(f, "quote"),
206            TokenKind::RangeOp => write!(f, ".."),
207            TokenKind::RBracket => write!(f, "]"),
208            TokenKind::RBrace => write!(f, "}}"),
209            TokenKind::RParen => write!(f, ")"),
210            TokenKind::Selector(selector) => write!(f, "{}", selector),
211            TokenKind::Self_ => write!(f, "self"),
212            TokenKind::SemiColon => write!(f, ";"),
213            TokenKind::StringLiteral(s) => write!(f, "{}", s),
214            TokenKind::StarEqual => write!(f, "*="),
215            TokenKind::Tab(n) => write!(f, "{}", "\t".repeat(*n)),
216            TokenKind::Try => write!(f, "try"),
217            TokenKind::Unquote => write!(f, "unquote"),
218            TokenKind::Catch => write!(f, "catch"),
219            TokenKind::While => write!(f, "while"),
220            TokenKind::Whitespace(n) => write!(f, "{}", " ".repeat(*n)),
221            TokenKind::LBrace => write!(f, "{{"),
222            TokenKind::Question => write!(f, "?"),
223            TokenKind::Var => write!(f, "var"),
224        }
225    }
226}
227#[cfg(test)]
228mod tests {
229    use super::*;
230    use rstest::rstest;
231
232    #[rstest]
233    #[case(
234        StringSegment::Text("hello".to_string(), Range::default()),
235        "hello"
236    )]
237    #[case(StringSegment::Expr(SmolStr::new("world"), Range::default()), "${world}")]
238    #[case(
239        StringSegment::Text("".to_string(), Range::default()),
240        ""
241    )]
242    #[case(StringSegment::Expr(SmolStr::new(""), Range::default()), "${}")]
243    fn string_segment_display_works(#[case] segment: StringSegment, #[case] expected: &str) {
244        assert_eq!(segment.to_string(), expected);
245    }
246}
mq_lang/lexer/token.rs

mq_lang/lexer/
token.rs
