aplang_lib/lexer/
token.rs

1use crate::parser::ast::{BinaryOp, LogicalOp, UnaryOp};
2use miette::{miette, LabeledSpan, SourceSpan};
3use std::collections::HashMap;
4use std::fmt;
5use std::sync::Arc;
6
7#[derive(Debug, Clone, PartialEq)]
8pub enum TokenType {
9    // Smart
10    SoftSemi,
11
12    // Single-char tokens
13    LeftParen,
14    RightParen,
15    LeftBracket,
16    RightBracket,
17    LeftBrace,
18    RightBrace,
19    Comma,
20    Dot,
21    Minus,
22    Plus,
23    Slash,
24    Star,
25
26    // Mixed
27    Arrow,
28    EqualEqual,
29    BangEqual,
30    Greater,
31    GreaterEqual,
32    Less,
33    LessEqual,
34
35    // Literals
36    Identifier,
37    Number,
38    StringLiteral,
39
40    // Keywords
41    Mod,
42    If,
43    Else,
44    Repeat,
45    Times,
46    Until,
47    For,
48    Each,
49    Continue,
50    Break,
51    In,
52    Procedure,
53    Return,
54    Not,
55    And,
56    Or,
57
58    // Keyword Literals
59    True,
60    False,
61    Null,
62
63    // Modules
64    Import,
65    Export,
66    From,
67
68    Eof,
69}
70
71#[derive(Debug, Clone)]
72pub struct Token {
73    pub token_type: TokenType,
74    pub lexeme: String,
75    pub literal: Option<LiteralValue>,
76    pub span: SourceSpan,
77    pub line_number: usize,
78    pub source: Arc<str>,
79}
80
81// Implement Display for Token
82impl fmt::Display for Token {
83    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
84        write!(f, "{} ", self.lexeme)
85    }
86}
87
88impl Token {
89    pub fn debug_many(tokens: &Vec<Token>) -> String {
90        let string: Vec<String> = tokens.iter().map(|t| format!("{t}")).collect();
91        format!("[{}]", string.join(", "))
92    }
93}
94
95impl Token {
96    pub fn is_soft_semi(&self) -> bool {
97        matches!(self.token_type, TokenType::SoftSemi)
98    }
99}
100
101pub fn get_keywords_hashmap() -> HashMap<&'static str, TokenType> {
102    use mapro::map;
103    use TokenType::*;
104    map! {
105        "mod" => Mod, "MOD" => Mod,
106        "if" => If, "IF" => If,
107        "else" => Else, "ELSE" => Else,
108        "repeat" => Repeat, "REPEAT" => Repeat,
109        "times" => Times, "TIMES" => Times,
110        "until" => Until, "UNTIL" => Until,
111        "for" => For, "FOR" => For,
112        "each" => Each, "EACH" => Each,
113        "continue" => Continue, "CONTINUE" => Continue,
114        "break" => Break, "BREAK" => Break,
115        "in" => In, "IN" => In,
116        "procedure" => Procedure, "PROCEDURE" => Procedure,
117        "return" => Return, "RETURN" => Return,
118        "not" => Not, "NOT" => Not,
119        "and" => And, "AND" => And,
120        "or" => Or, "OR" => Or,
121        "true" => True, "TRUE" => True,
122        "false" => False, "FALSE" => False,
123        "null" => Null, "NULL" => Null,
124        "import" => Import, "IMPORT" => Import,
125        "export" => Export, "EXPORT" => Export,
126        "from" => From, "FROM" => From,
127    }
128}
129
130impl Token {
131    pub fn token_type(&self) -> &TokenType {
132        &self.token_type
133    }
134    pub fn label(&self, label: impl Into<String>) -> LabeledSpan {
135        LabeledSpan::at(self.span, label)
136    }
137
138    pub fn span_to_label(&self, other: SourceSpan, label: impl Into<String>) -> LabeledSpan {
139        LabeledSpan::at(self.span_to(other), label)
140    }
141
142    pub fn span(&self) -> SourceSpan {
143        self.span
144    }
145
146    pub fn span_to(&self, other: SourceSpan) -> SourceSpan {
147        join_spans(self.span(), other)
148    }
149
150    pub fn span_until(&self, other: SourceSpan) -> SourceSpan {
151        span_between(self.span(), other)
152    }
153
154    pub fn span_to_token(&self, other: &Token) -> SourceSpan {
155        self.span_to(other.span())
156    }
157
158    pub fn span_until_token(&self, other: &Token) -> SourceSpan {
159        self.span_until(other.span())
160    }
161}
162
163pub fn join_spans(left: SourceSpan, right: SourceSpan) -> SourceSpan {
164    let length = right.offset() - left.offset() + right.len();
165    SourceSpan::from(left.offset()..length)
166}
167
168pub fn span_between(left: SourceSpan, right: SourceSpan) -> SourceSpan {
169    SourceSpan::from(left.offset() + left.len()..right.offset())
170}
171
172impl Token {
173    pub fn to_binary_op(&self) -> miette::Result<BinaryOp> {
174        match self.token_type {
175            TokenType::EqualEqual => Ok(BinaryOp::EqualEqual),
176            TokenType::BangEqual => Ok(BinaryOp::NotEqual),
177            TokenType::Less => Ok(BinaryOp::Less),
178            TokenType::LessEqual => Ok(BinaryOp::LessEqual),
179            TokenType::Greater => Ok(BinaryOp::Greater),
180            TokenType::GreaterEqual => Ok(BinaryOp::GreaterEqual),
181            TokenType::Plus => Ok(BinaryOp::Plus),
182            TokenType::Minus => Ok(BinaryOp::Minus),
183            TokenType::Star => Ok(BinaryOp::Star),
184            TokenType::Slash => Ok(BinaryOp::Slash),
185            TokenType::Mod => Ok(BinaryOp::Modulo),
186            // todo: improve this message
187            _ => Err(miette!(
188                "Conversion to Binary Op Error, Token is not binary Op"
189            )),
190        }
191    }
192
193    pub fn to_unary_op(&self) -> miette::Result<UnaryOp> {
194        match self.token_type {
195            TokenType::Minus => Ok(UnaryOp::Minus),
196            TokenType::Not => Ok(UnaryOp::Not),
197            // todo: improve this message
198            _ => Err(miette!(
199                "Conversion to Binary Unary Error, Token is not Unary op"
200            )),
201        }
202    }
203
204    pub fn to_logical_op(&self) -> miette::Result<LogicalOp> {
205        match self.token_type {
206            TokenType::Or => Ok(LogicalOp::Or),
207            TokenType::And => Ok(LogicalOp::And),
208            // todo: improve this message
209            _ => Err(miette!(
210                "Conversion to Binary Logical Error, Token is not Logical op"
211            )),
212        }
213    }
214}
215
216#[derive(Debug, Clone, PartialEq)]
217pub enum LiteralValue {
218    Number(f64),
219    String(String),
220}
221
222impl TryInto<f64> for LiteralValue {
223    type Error = String;
224
225    fn try_into(self) -> miette::Result<f64, Self::Error> {
226        let Self::Number(num) = self else {
227            return Err(
228                "Trying to convert to number when literal is not of type number".to_string(),
229            );
230        };
231
232        Ok(num)
233    }
234}
235
236impl TryInto<String> for LiteralValue {
237    type Error = String;
238
239    fn try_into(self) -> miette::Result<String, Self::Error> {
240        let Self::String(string) = self else {
241            return Err(
242                "Trying to convert to string when literal is not of type string".to_string(),
243            );
244        };
245
246        Ok(string)
247    }
248}
aplang_lib/lexer/token.rs

aplang_lib/lexer/
token.rs
