jist/
token_type.rs

1/*
2* This file contains the token types used by the tokenizer, more can be added as needed to process
3* a variety of syntax expressions
4*/
5
6pub mod token_types {
7
8    #[derive(Debug, Clone)]
9    pub enum TokenTypes {
10        /*
11         * A simple numeric value
12         */
13        Int,
14        /*
15         * A simple string value
16         */
17        String,
18        /*
19         * A simple character value
20         */
21        Char,
22        /*
23         * the = operator
24         */
25        AssignmentOperator,
26
27        /*
28         * either true or false
29         */
30        Bool,
31        /*
32         * A function argument
33         */
34        FunctionArguments,
35        /*
36         * An operator (+,-,/,*)
37         */
38        Operator,
39        /*
40         * (
41         */
42        LeftParenthesis,
43        /*
44         * )
45         */
46        RightParenthesis,
47        /*
48         * func
49         */
50        Function {
51            name: String,
52            return_type: String,
53            arguments: Vec<(String, String, String)>,
54            block: Vec<String>,
55        },
56        /*
57         * funcname()
58         */
59        FunctionCall,
60        /*
61         * ',' Comma used to separate function arguments
62         */
63        ArgumentSeparator,
64        /*
65         * a = 2
66         */
67        VariableCall,
68        /*
69         * values within () in a function call
70         */
71        FunctionCallArguments,
72        /*
73         * 'let' used to declare a variable
74         */
75        Assignment,
76        /*
77         * '}'
78         */
79        RightCurly,
80        /*
81         * '{'
82         */
83        LeftCurly,
84        /*
85         *let
86         */
87        Variable,
88        /*
89         * ':'
90         */
91        VarTypeAssignment,
92        /*
93         * '->'
94         */
95        ReturnTypeAssignment,
96        /*
97         * ; semicolon
98         */
99        SemiColon,
100        /*
101         * // or /* */
102         */
103        Comment,
104        /*
105         * 1.102
106         */
107        Float,
108        /*
109         *   Collection
110         */
111        Collection {
112            name: String,
113            collection_type: String,
114            stored_value_type_single: String,
115            stored_value_type_tuple: (String, String),
116        },
117        /*
118         * if statement is simply a object name e.g. foo it is recognized as a call to an object,
119         * var, collection, etc
120         */
121        ObjectCall {
122            name: String,
123        },
124        /*
125         * [
126         */
127        LeftBracket,
128        /*
129         * ]
130         */
131        RightBracket,
132        /*
133         * =>
134         */
135        FatArrow,
136        /*
137        Used as a bad return value
138        */
139        None,
140
141        /*
142         * used for dot notation eg obj.method()
143         */
144        Dot {
145            object: String,
146            method: String,
147        },
148
149        /*
150         * If statement
151         */
152        If {
153            statement: String,
154        },
155        /*
156         * Else statement
157         */
158        Else,
159        /*
160         * Elif statement
161         */
162        Elif {
163            statement: String,
164        },
165        /*
166         * While statement
167         */
168        While {
169            statement: String,
170            block: Vec<String>,
171        },
172        /*
173         * For statement
174         */
175        For {
176            variable: String,
177            iterable: (i32, i32),
178            block: Vec<String>,
179        },
180        /*
181         * Break statement
182         */
183        Break,
184        /*
185         * Continue statement
186         */
187        Continue,
188        /*
189         * Try statement
190         */
191        Try {
192            block: Vec<String>,
193        },
194        /*
195         * Catch statement
196         */
197        Catch {
198            block: Vec<String>,
199        },
200        /*
201         * Finally statement
202         */
203        Finally {
204            block: Vec<String>,
205        },
206
207        /*
208         * !
209         * */
210        Not,
211
212        /*
213         * Used for a return statement
214         */
215        ReturnStatement {
216            value: String,
217        },
218    }
219
220    impl PartialEq for TokenTypes {
221        fn eq(&self, other: &Self) -> bool {
222            match (self, other) {
223                (
224                    TokenTypes::Dot {
225                        object: ref a,
226                        method: ref b,
227                    },
228                    TokenTypes::Dot {
229                        object: ref c,
230                        method: ref d,
231                    },
232                ) => a == c && b == d,
233                (TokenTypes::FunctionCallArguments, TokenTypes::FunctionCallArguments) => true,
234                (
235                    TokenTypes::ObjectCall { name: ref name1 },
236                    TokenTypes::ObjectCall { name: ref name2 },
237                ) => name1 == name2,
238                (TokenTypes::SemiColon, TokenTypes::SemiColon) => true,
239                (TokenTypes::Int, TokenTypes::Int) => true,
240                (TokenTypes::Float, TokenTypes::Float) => true,
241                (TokenTypes::String, TokenTypes::String) => true,
242                (TokenTypes::Char, TokenTypes::Char) => true,
243                (TokenTypes::Operator, TokenTypes::Operator) => true,
244                (TokenTypes::AssignmentOperator, TokenTypes::AssignmentOperator) => true,
245                (TokenTypes::LeftParenthesis, TokenTypes::LeftParenthesis) => true,
246                (TokenTypes::RightParenthesis, TokenTypes::RightParenthesis) => true,
247                (TokenTypes::FunctionCall, TokenTypes::FunctionCall) => true,
248                (TokenTypes::VariableCall, TokenTypes::VariableCall) => true,
249                (TokenTypes::ArgumentSeparator, TokenTypes::ArgumentSeparator) => true,
250                (TokenTypes::Assignment, TokenTypes::Assignment) => true,
251                (TokenTypes::VarTypeAssignment, TokenTypes::VarTypeAssignment) => true,
252                (TokenTypes::RightCurly, TokenTypes::RightCurly) => true,
253                (TokenTypes::LeftCurly, TokenTypes::LeftCurly) => true,
254                (TokenTypes::ReturnTypeAssignment, TokenTypes::ReturnTypeAssignment) => true,
255                (
256                    TokenTypes::ReturnStatement { value },
257                    TokenTypes::ReturnStatement { value: val },
258                ) => value == val,
259                (TokenTypes::Variable, TokenTypes::Variable) => true,
260
261                (
262                    TokenTypes::Function {
263                        name: ref name_a,
264                        return_type: ref return_a,
265                        arguments: ref args_a,
266                        ..
267                    },
268                    TokenTypes::Function {
269                        name: ref name_b,
270                        return_type: ref return_b,
271                        arguments: ref args_b,
272                        ..
273                    },
274                ) => name_a == name_b && return_a == return_b,
275
276                (
277                    TokenTypes::Collection {
278                        name: ref name_a,
279                        collection_type: ref type_a,
280                        stored_value_type_single: ref stored_a,
281                        stored_value_type_tuple: ref _tuple_a,
282                    },
283                    TokenTypes::Collection {
284                        name: ref name_b,
285                        collection_type: ref type_b,
286                        stored_value_type_single: ref stored_b,
287                        stored_value_type_tuple: ref _tuple_b,
288                    },
289                ) => name_a == name_b && type_a == type_b && stored_a == stored_b,
290
291                (TokenTypes::Comment, TokenTypes::Comment) => true,
292                (TokenTypes::Bool, TokenTypes::Bool) => true,
293                (TokenTypes::LeftBracket, TokenTypes::LeftBracket) => true,
294                (TokenTypes::RightBracket, TokenTypes::RightBracket) => true,
295                (TokenTypes::FatArrow, TokenTypes::FatArrow) => true,
296                (TokenTypes::None, TokenTypes::None) => true,
297                (
298                    TokenTypes::If {
299                        statement: ref statement_a,
300                    },
301                    TokenTypes::If {
302                        statement: ref statement_b,
303                    },
304                ) => statement_a == statement_b,
305                (TokenTypes::Else, TokenTypes::Else) => true,
306                (
307                    TokenTypes::Elif {
308                        statement: ref statement_a,
309                    },
310                    TokenTypes::Elif {
311                        statement: ref statement_b,
312                    },
313                ) => statement_a == statement_b,
314                (TokenTypes::Break, TokenTypes::Break) => true,
315                (TokenTypes::Continue, TokenTypes::Continue) => true,
316                (
317                    TokenTypes::Try {
318                        block: ref statement_a,
319                        ..
320                    },
321                    TokenTypes::Try {
322                        block: ref statement_b,
323                        ..
324                    },
325                ) => statement_a == statement_b,
326                (
327                    TokenTypes::Catch { block: ref block_a },
328                    TokenTypes::Catch { block: ref block_b },
329                ) => block_a == block_b,
330                (
331                    TokenTypes::Finally { block: ref block_a },
332                    TokenTypes::Finally { block: ref block_b },
333                ) => block_a == block_b,
334                (TokenTypes::Not, TokenTypes::Not) => true,
335                (
336                    TokenTypes::While {
337                        statement: ref a, ..
338                    },
339                    TokenTypes::While {
340                        statement: ref b, ..
341                    },
342                ) => a == b,
343                (
344                    TokenTypes::For {
345                        variable: ref a,
346                        iterable: ref a2,
347                        ..
348                    },
349                    TokenTypes::For {
350                        variable: ref b,
351                        iterable: ref b2,
352                        ..
353                    },
354                ) => a == b && a2 == b2,
355                _ => false,
356            }
357        }
358    }
359
360    impl Eq for TokenTypes {}
361
362    impl TokenTypes {
363        pub fn to_string(&self) -> String {
364            match self {
365                TokenTypes::ObjectCall { name } => format!("Object Call: name: {}", name),
366                TokenTypes::Dot { object, method } => format!("Dot: {}.{}", object, method),
367                TokenTypes::Function {
368                    name,
369                    return_type,
370                    arguments,
371                    block,
372                } => {
373                    let mut arguments_str = String::new();
374                    for arg in arguments {
375                        arguments_str.push_str(&format!("{:?} ", arg));
376                    }
377
378                    format!(
379                        "Function: {} {} {:?}, {:?}",
380                        name, return_type, arguments_str, block
381                    )
382                }
383                TokenTypes::Not => "Not".to_string(),
384                TokenTypes::Else => "Else".to_string(),
385                TokenTypes::Elif { statement } => format!("Elif: {}", statement),
386                TokenTypes::If { statement } => format!("If: {}", statement),
387                TokenTypes::While { statement, block } => {
388                    format!("While: {}, Block: {:?}", statement, block)
389                }
390                TokenTypes::For {
391                    variable,
392                    iterable,
393                    block,
394                } => {
395                    format!(
396                        "For: Var: {}, Iter: {:?}, Block: {:?}",
397                        variable, iterable, block
398                    )
399                }
400                TokenTypes::Break => "Break".to_string(),
401                TokenTypes::Continue => "Continue".to_string(),
402                TokenTypes::Try { block } => "Try".to_string(),
403                TokenTypes::Catch { block } => "Catch".to_string(),
404                TokenTypes::Finally { block } => "Finally".to_string(),
405                TokenTypes::FatArrow => "FatArrow".to_string(),
406                TokenTypes::FunctionCallArguments => "FunctionCallArguments".to_string(),
407                TokenTypes::Float => "Float".to_string(),
408                TokenTypes::SemiColon => "SemiColon".to_string(),
409                TokenTypes::FunctionArguments => "FunctionArguments".to_string(),
410                TokenTypes::Int => "Int".to_string(),
411                TokenTypes::String => "String".to_string(),
412                TokenTypes::Char => "Char".to_string(),
413                TokenTypes::Operator => "Operator".to_string(),
414                TokenTypes::AssignmentOperator => "AssignmentOperator".to_string(),
415                TokenTypes::Bool => "Bool".to_string(),
416                TokenTypes::LeftParenthesis => "LeftParenthesis".to_string(),
417                TokenTypes::RightParenthesis => "RightParenthesis".to_string(),
418                TokenTypes::FunctionCall => "FunctionCall".to_string(),
419                TokenTypes::Variable => "Variable".to_string(),
420                TokenTypes::VariableCall => "VariableCall".to_string(),
421                TokenTypes::ArgumentSeparator => "ArgumentSeparator".to_string(),
422                TokenTypes::Assignment => "Assignment".to_string(),
423                TokenTypes::VarTypeAssignment => "VarTypeAssignment".to_string(),
424                TokenTypes::RightCurly => "RightCurly".to_string(),
425                TokenTypes::Collection {
426                    name,
427                    collection_type,
428                    stored_value_type_single,
429                    stored_value_type_tuple,
430                } => {
431                    format!(
432                        "Collection: {} {} {} {:?}",
433                        name, collection_type, stored_value_type_single, stored_value_type_tuple
434                    )
435                }
436                TokenTypes::LeftCurly => "LeftCurly".to_string(),
437                TokenTypes::ReturnTypeAssignment => "ReturnTypeAssignment".to_string(),
438                TokenTypes::Comment => "Comment".to_string(),
439                TokenTypes::RightBracket => "RightBracket".to_string(),
440                TokenTypes::LeftBracket => "LeftBracket".to_string(),
441                TokenTypes::ReturnStatement { value } => format!("ReturnStatement: {}", value),
442                TokenTypes::None => "None".to_string(),
443            }
444        }
445    }
446}
jist/token_type.rs

jist/
token_type.rs
