mq_lang/lexer/
token.rs

1use std::fmt::{self, Display, Formatter};
2
3use itertools::Itertools;
4use smol_str::SmolStr;
5
6use crate::{ArenaId, module::ModuleId, number::Number, range::Range};
7#[cfg(feature = "ast-json")]
8use serde::{Deserialize, Serialize};
9
10#[cfg_attr(feature = "ast-json", derive(Serialize, Deserialize))]
11#[derive(Debug, Clone, PartialOrd, PartialEq, Ord, Eq)]
12pub enum StringSegment {
13    Text(String, Range),
14    Expr(SmolStr, Range),
15}
16
17impl Display for StringSegment {
18    fn fmt(&self, f: &mut Formatter<'_>) -> Result<(), fmt::Error> {
19        match self {
20            StringSegment::Text(text, _) => write!(f, "{}", text),
21            StringSegment::Expr(expr, _) => write!(f, "${{{}}}", expr),
22        }
23    }
24}
25
26fn default_module_id() -> ModuleId {
27    ArenaId::new(0)
28}
29
30#[cfg_attr(feature = "ast-json", derive(Serialize, Deserialize))]
31#[derive(PartialEq, Eq, PartialOrd, Ord, Debug, Clone)]
32pub struct Token {
33    pub range: Range,
34    pub kind: TokenKind,
35    #[cfg_attr(
36        feature = "ast-json",
37        serde(skip_serializing, skip_deserializing, default = "default_module_id")
38    )]
39    pub module_id: ModuleId,
40}
41
42#[cfg_attr(feature = "ast-json", derive(Serialize, Deserialize))]
43#[derive(PartialEq, Eq, PartialOrd, Ord, Debug, Clone)]
44/// Represents the kind of a token in the mq language.
45///
46/// TokenKind variants are sorted alphabetically for maintainability.
47pub enum TokenKind {
48    And,
49    Convert,
50    Asterisk,
51    BoolLiteral(bool),
52    Break,
53    Catch,
54    Coalesce,
55    Colon,
56    DoubleColon,
57    DoubleSlashEqual,
58    Comma,
59    Comment(String),
60    Continue,
61    Def,
62    Do,
63    Elif,
64    Else,
65    End,
66    Env(SmolStr),
67    Eof,
68    Equal,
69    EqEq,
70    Fn,
71    Foreach,
72    Gt,
73    Gte,
74    Ident(SmolStr),
75    If,
76    Include,
77    InterpolatedString(Vec<StringSegment>),
78    Import,
79    LBrace,
80    LBracket,
81    Let,
82    LeftShift,
83    Loop,
84    Lt,
85    Lte,
86    Macro,
87    Match,
88    Module,
89    Minus,
90    MinusEqual,
91    NeEq,
92    NewLine,
93    Nodes,
94    None,
95    Not,
96    NumberLiteral(Number),
97    Or,
98    Percent,
99    PercentEqual,
100    Pipe,
101    PipeEqual,
102    Plus,
103    PlusEqual,
104    Question,
105    Quote,
106    RBrace,
107    DoubleDot,
108    RBracket,
109    RightShift,
110    RParen,
111    Selector(SmolStr),
112    Self_,
113    SemiColon,
114    Slash,
115    SlashEqual,
116    StringLiteral(String),
117    StarEqual,
118    Tab(usize),
119    TildeEqual,
120    Try,
121    Unquote,
122    Whitespace(usize),
123    While,
124    LParen,
125    Var,
126}
127
128impl Token {
129    pub fn new(kind: TokenKind) -> Self {
130        Self {
131            kind,
132            range: Range::default(),
133            module_id: default_module_id(),
134        }
135    }
136
137    #[inline(always)]
138    pub fn is_eof(&self) -> bool {
139        matches!(self.kind, TokenKind::Eof)
140    }
141
142    #[inline(always)]
143    pub fn is_selector(&self) -> bool {
144        matches!(self.kind, TokenKind::Selector(_))
145    }
146}
147
148impl Display for Token {
149    fn fmt(&self, f: &mut Formatter<'_>) -> Result<(), fmt::Error> {
150        write!(f, "{}", self.kind)
151    }
152}
153
154impl Display for TokenKind {
155    fn fmt(&self, f: &mut Formatter<'_>) -> Result<(), fmt::Error> {
156        match &self {
157            TokenKind::And => write!(f, "&&"),
158            TokenKind::Convert => write!(f, "@"),
159            TokenKind::Or => write!(f, "||"),
160            TokenKind::Not => write!(f, "!"),
161            TokenKind::Asterisk => write!(f, "*"),
162            TokenKind::BoolLiteral(b) => write!(f, "{}", b),
163            TokenKind::Break => write!(f, "break"),
164            TokenKind::Colon => write!(f, ":"),
165            TokenKind::Comma => write!(f, ","),
166            TokenKind::Continue => write!(f, "continue"),
167            TokenKind::Coalesce => write!(f, "??"),
168            TokenKind::Comment(comment) => write!(f, "# {}", comment.trim()),
169            TokenKind::Def => write!(f, "def"),
170            TokenKind::Do => write!(f, "do"),
171            TokenKind::DoubleColon => write!(f, "::"),
172            TokenKind::DoubleSlashEqual => write!(f, "//="),
173            TokenKind::Elif => write!(f, "elif"),
174            TokenKind::Else => write!(f, "else"),
175            TokenKind::End => write!(f, "end"),
176            TokenKind::Env(env) => write!(f, "${}", env),
177            TokenKind::Eof => write!(f, ""),
178            TokenKind::Equal => write!(f, "="),
179            TokenKind::EqEq => write!(f, "=="),
180            TokenKind::Fn => write!(f, "fn"),
181            TokenKind::Foreach => write!(f, "foreach"),
182            TokenKind::Ident(ident) => write!(f, "{}", ident),
183            TokenKind::If => write!(f, "if"),
184            TokenKind::Include => write!(f, "include"),
185            TokenKind::Import => write!(f, "import"),
186            TokenKind::InterpolatedString(segments) => {
187                write!(f, "{}", segments.iter().join(""))
188            }
189            TokenKind::Lt => write!(f, "<"),
190            TokenKind::Lte => write!(f, "<="),
191            TokenKind::Gt => write!(f, ">"),
192            TokenKind::Gte => write!(f, ">="),
193            TokenKind::LBracket => write!(f, "["),
194            TokenKind::LParen => write!(f, "("),
195            TokenKind::LeftShift => write!(f, "<<"),
196            TokenKind::Let => write!(f, "let"),
197            TokenKind::Loop => write!(f, "loop"),
198            TokenKind::Macro => write!(f, "macro"),
199            TokenKind::Match => write!(f, "match"),
200            TokenKind::Module => write!(f, "module"),
201            TokenKind::Minus => write!(f, "-"),
202            TokenKind::MinusEqual => write!(f, "-="),
203            TokenKind::Slash => write!(f, "/"),
204            TokenKind::SlashEqual => write!(f, "/="),
205            TokenKind::Percent => write!(f, "%"),
206            TokenKind::PercentEqual => write!(f, "%="),
207            TokenKind::NeEq => write!(f, "!="),
208            TokenKind::NewLine => writeln!(f),
209            TokenKind::Nodes => write!(f, "nodes"),
210            TokenKind::None => write!(f, "None"),
211            TokenKind::NumberLiteral(n) => write!(f, "{}", n),
212            TokenKind::Plus => write!(f, "+"),
213            TokenKind::PlusEqual => write!(f, "+="),
214            TokenKind::Pipe => write!(f, "|"),
215            TokenKind::PipeEqual => write!(f, "|="),
216            TokenKind::Quote => write!(f, "quote"),
217            TokenKind::DoubleDot => write!(f, ".."),
218            TokenKind::RBracket => write!(f, "]"),
219            TokenKind::RightShift => write!(f, ">>"),
220            TokenKind::RBrace => write!(f, "}}"),
221            TokenKind::RParen => write!(f, ")"),
222            TokenKind::Selector(selector) => write!(f, "{}", selector),
223            TokenKind::Self_ => write!(f, "self"),
224            TokenKind::SemiColon => write!(f, ";"),
225            TokenKind::StringLiteral(s) => write!(f, "{}", s),
226            TokenKind::StarEqual => write!(f, "*="),
227            TokenKind::Tab(n) => write!(f, "{}", "\t".repeat(*n)),
228            TokenKind::TildeEqual => write!(f, "=~"),
229            TokenKind::Try => write!(f, "try"),
230            TokenKind::Unquote => write!(f, "unquote"),
231            TokenKind::Catch => write!(f, "catch"),
232            TokenKind::While => write!(f, "while"),
233            TokenKind::Whitespace(n) => write!(f, "{}", " ".repeat(*n)),
234            TokenKind::LBrace => write!(f, "{{"),
235            TokenKind::Question => write!(f, "?"),
236            TokenKind::Var => write!(f, "var"),
237        }
238    }
239}
240
241#[cfg(test)]
242mod tests {
243    use super::*;
244    use rstest::rstest;
245
246    #[rstest]
247    #[case(
248        StringSegment::Text("hello".to_string(), Range::default()),
249        "hello"
250    )]
251    #[case(StringSegment::Expr(SmolStr::new("world"), Range::default()), "${world}")]
252    #[case(
253        StringSegment::Text("".to_string(), Range::default()),
254        ""
255    )]
256    #[case(StringSegment::Expr(SmolStr::new(""), Range::default()), "${}")]
257    fn string_segment_display_works(#[case] segment: StringSegment, #[case] expected: &str) {
258        assert_eq!(segment.to_string(), expected);
259    }
260}
mq_lang/lexer/token.rs

mq_lang/lexer/
token.rs
