astray_core/token/
mod.rs

1use std::cmp::max;
2
3use crate::{ConsumableToken, Parsable, ParseError, TokenIter};
4
5#[derive(PartialEq, Default, Debug, Clone)]
6pub struct LiteralStringValue {
7    value: String,
8}
9
10impl From<String> for LiteralStringValue {
11    fn from(s: String) -> Self {
12        LiteralStringValue { value: s }
13    }
14}
15
16#[derive(PartialEq, Default, Debug, Clone)]
17pub struct LiteralIntValue {
18    value: String,
19}
20
21impl From<String> for LiteralIntValue {
22    fn from(s: String) -> Self {
23        LiteralIntValue { value: s }
24    }
25}
26
27#[derive(PartialEq, Default, Debug, Clone)]
28pub struct IdentifierValue {
29    value: String,
30}
31
32impl From<String> for IdentifierValue {
33    fn from(s: String) -> Self {
34        IdentifierValue { value: s }
35    }
36}
37
38#[derive(Debug, Clone, PartialEq, Eq)]
39pub enum Token {
40    // Operations
41    Assign,
42    Plus,
43    Minus,
44    Mult,
45    Div,
46    // KeyWords and Literals
47    KInt,
48    KFloat,
49    KReturn,
50    // Literals
51    LiteralString(String),
52    LiteralInt(u32),
53    //Identifier
54    Identifier(String),
55    // Delimeters
56    RCurly,
57    LCurly,
58    RBracket,
59    LBracket,
60    RParen,
61    LParen,
62    Comma,
63    //Punctuation
64    SemiColon,
65    // meta
66    INVALID,
67    EMPTY,
68}
69
70impl Token {
71    pub fn from_regex_result(&self, input: String) -> (Token, usize) {
72        let token = match *self {
73            Token::LiteralString(_) => Token::LiteralString(input[1..input.len() - 1].to_string()), // remove  quotes around string
74            Token::Identifier(_) => Token::Identifier(input.clone()), // remove  quotes around string
75            Token::LiteralInt(_) => {
76                let s: u32 = input.parse().unwrap();
77                Token::LiteralInt(s)
78            }
79            _ => (*self).clone(),
80        };
81        (token, input.len())
82    }
83}
84
85#[macro_export]
86macro_rules! t {
87    (,) => {
88        Token::Comma
89    };
90    (, def) => {
91        ","
92    };
93    (;) => {
94        Token::SemiColon
95    };
96    (; def) => {
97        ";"
98    };
99    (=) => {
100        Token::Assign
101    };
102    (= def) => {
103        "="
104    };
105    (+) => {
106        Token::Plus
107    };
108    (+ def) => {
109        "+"
110    };
111    (-) => {
112        Token::Minus
113    };
114    (- def) => {
115        "-"
116    };
117    (*) => {
118        Token::Mult
119    };
120    (* def) => {
121        "*"
122    };
123    (/) => {
124        Token::Div
125    };
126    (/ def) => {
127        "/"
128    };
129    (int) => {
130        Token::KInt
131    };
132    (float) => {
133        Token::KFloat
134    };
135    (int def) => {
136        "int"
137    };
138    (return) => {
139        Token::KReturn
140    };
141    (return def) => {
142        "return"
143    };
144
145    (litstr) => {
146        Token::LiteralString("DEFAULT_LITERAL_STRING")
147    };
148    (litint) => {
149        Token::LiteralInt(0)
150    };
151    (ident) => {
152        Token::Identifier("DEFAULT_IDENTIFIER".to_string())
153    };
154
155    (litstr $value:expr) => {
156        Token::LiteralString($value.to_string())
157    };
158    (litint $value:expr) => {
159        Token::LiteralInt($value)
160    };
161    (ident $value:expr) => {
162        Token::Identifier($value.to_string())
163    };
164
165    (r_paren) => {
166        Token::RParen
167    };
168    (r_paren def) => {
169        ")"
170    };
171    (l_paren) => {
172        Token::LParen
173    };
174    (l_paren def) => {
175        "("
176    };
177    (r_curly) => {
178        Token::RCurly
179    };
180    (r_curly def) => {
181        "}"
182    };
183    (l_curly) => {
184        Token::LCurly
185    };
186    (l_curly def) => {
187        "{"
188    };
189    (r_bracket) => {
190        Token::RBracket
191    };
192    (r_bracket def) => {
193        "]"
194    };
195    (l_bracket) => {
196        Token::LBracket
197    };
198    (l_bracket def) => {
199        "["
200    };
201    (empty) => {
202        Token::EMPTY
203    };
204    (invalid) => {
205        Token::INVALID
206    };
207}
208
209impl ConsumableToken for Token {}
210
211impl Parsable<Token> for Token {
212    fn parse(iter: &mut TokenIter<Token>) -> Result<Self, ParseError<Token>>
213    where
214        Self: Sized,
215    {
216        match iter.consume() {
217            Some(token) => Ok(token),
218            None => Err(ParseError::no_more_tokens::<Token>(iter.current)),
219        }
220    }
221
222    fn parse_if_match<F: Fn(&Token) -> bool>(
223        iter: &mut TokenIter<Token>,
224        matches: F,
225        pattern: Option<&'static str>
226    ) -> Result<Self, ParseError<Token>>
227    where
228        Self: Sized,
229    {
230        // TODO: find a way to express this that doesn't need comparison:
231        // this introduces extra instructions every time a token is parsed
232        iter.try_do(|token_iter| match token_iter.consume() {
233            Some(ref found) if matches(found) => Ok(found.clone()),
234            Some(ref found) => Err(ParseError::parsed_but_unmatching::<Token>(
235                if token_iter.current == 0 {
236                    token_iter.current
237                } else {
238                    token_iter.current - 1
239                },
240                found,
241                pattern
242            )),
243            _ => Err(ParseError::no_more_tokens::<Token>(if token_iter.current == 0 {
244                token_iter.current
245            } else {
246                token_iter.current - 1
247            })),
248        })
249    }
250}
astray_core/token/mod.rs

astray_core/token/
mod.rs
