devalang_core/core/parser/
driver.rs

1use crate::core::{
2    lexer::token::{Token, TokenKind},
3    parser::{
4        handler::{
5            arrow_call::parse_arrow_call,
6            at::parse_at_token,
7            bank::parse_bank_token,
8            condition::parse_condition_token,
9            dot::parse_dot_token,
10            identifier::{function::parse_function_token, parse_identifier_token},
11            loop_::parse_loop_token,
12            tempo::parse_tempo_token,
13        },
14        statement::Statement,
15    },
16    shared::value::Value,
17    store::global::GlobalStore,
18};
19
20#[derive(Debug, Clone, PartialEq)]
21pub struct Parser {
22    pub resolve_modules: bool,
23    pub tokens: Vec<Token>,
24    pub token_index: usize,
25    pub current_module: String,
26    pub previous: Option<Token>,
27}
28
29impl Parser {
30    pub fn new() -> Self {
31        Parser {
32            resolve_modules: false,
33            tokens: Vec::new(),
34            token_index: 0,
35            current_module: String::new(),
36            previous: None,
37        }
38    }
39
40    pub fn set_current_module(&mut self, module_path: String) {
41        self.current_module = module_path;
42    }
43
44    pub fn advance(&mut self) -> Option<&Token> {
45        if self.is_eof() {
46            return None;
47        }
48
49        self.previous = self.tokens.get(self.token_index).cloned();
50        self.token_index += 1;
51
52        self.tokens.get(self.token_index - 1)
53    }
54
55    pub fn peek_is(&self, expected: &str) -> bool {
56        self.peek().map_or(false, |t| t.lexeme == expected)
57    }
58
59    pub fn peek_nth(&self, n: usize) -> Option<&Token> {
60        if self.token_index + n < self.tokens.len() {
61            self.tokens.get(self.token_index + n)
62        } else {
63            None
64        }
65    }
66
67    pub fn peek_nth_kind(&self, n: usize) -> Option<TokenKind> {
68        self.peek_nth(n).map(|t| t.kind.clone())
69    }
70
71    pub fn advance_if(&mut self, kind: TokenKind) -> bool {
72        if self.match_token(kind) { true } else { false }
73    }
74
75    pub fn match_token(&mut self, kind: TokenKind) -> bool {
76        if let Some(tok) = self.peek() {
77            if tok.kind == kind {
78                self.advance();
79                return true;
80            }
81        }
82        false
83    }
84
85    pub fn previous_clone(&self) -> Option<Token> {
86        self.previous.clone()
87    }
88
89    pub fn parse_block(
90        &self,
91        tokens: Vec<Token>,
92        global_store: &mut GlobalStore,
93    ) -> Vec<Statement> {
94        let mut inner_parser = Parser {
95            resolve_modules: self.resolve_modules,
96            tokens,
97            token_index: 0,
98            current_module: self.current_module.clone(),
99            previous: None,
100        };
101
102        inner_parser.parse_tokens(inner_parser.tokens.clone(), global_store)
103    }
104
105    pub fn parse_tokens(
106        &mut self,
107        tokens: Vec<Token>,
108        global_store: &mut GlobalStore,
109    ) -> Vec<Statement> {
110        // Filter out Whitespace tokens but keep Newline tokens because some constructs (e.g., print ...)
111        // rely on end-of-line semantics.
112        self.tokens = tokens
113            .into_iter()
114            .filter(|t| t.kind != TokenKind::Whitespace)
115            .collect();
116        self.token_index = 0;
117
118        let mut statements = Vec::new();
119
120        while !self.is_eof() {
121            let token = match self.peek() {
122                Some(t) => t.clone(),
123                None => {
124                    break;
125                }
126            };
127
128            if token.kind == TokenKind::Newline {
129                self.advance();
130                continue;
131            }
132
133            let statement = match &token.kind {
134                TokenKind::At => parse_at_token(self, global_store),
135                TokenKind::Identifier => {
136                    if let Some(next) = self.peek_nth(1).cloned() {
137                        if next.kind == TokenKind::Arrow {
138                            parse_arrow_call(self, global_store)
139                        } else {
140                            parse_identifier_token(self, global_store)
141                        }
142                    } else {
143                        parse_identifier_token(self, global_store)
144                    }
145                }
146                TokenKind::Dot => parse_dot_token(self, global_store),
147                TokenKind::Tempo => parse_tempo_token(self, global_store),
148                TokenKind::Bank => parse_bank_token(self, global_store),
149                TokenKind::Loop => parse_loop_token(self, global_store),
150                TokenKind::If => parse_condition_token(self, global_store),
151                TokenKind::Function => parse_function_token(self, global_store),
152                TokenKind::On => crate::core::parser::handler::identifier::on::parse_on_token(self, global_store),
153                TokenKind::Emit => crate::core::parser::handler::identifier::emit::parse_emit_token(self, token.clone(), global_store),
154
155                | TokenKind::Else // Ignore else, already handled in `parse_condition_token`
156                | TokenKind::Comment
157                | TokenKind::Equals
158                | TokenKind::Colon
159                | TokenKind::Number
160                | TokenKind::String
161                | TokenKind::LBrace
162                | TokenKind::RBrace
163                | TokenKind::Comma
164                | TokenKind::Dedent
165                | TokenKind::Indent => {
166                    self.advance();
167                    continue;
168                }
169
170                TokenKind::EOF => {
171                    break;
172                }
173
174                _ => {
175                    self.advance();
176                    Statement::unknown_from_token(&token)
177                }
178            };
179
180            statements.push(statement);
181        }
182
183        statements
184    }
185
186    pub fn check_token(&self, kind: TokenKind) -> bool {
187        self.peek().map_or(false, |t| t.kind == kind)
188    }
189
190    pub fn peek_kind(&self) -> Option<TokenKind> {
191        self.peek().map(|t| t.kind.clone())
192    }
193
194    pub fn parse_map_value(&mut self) -> Option<Value> {
195        if !self.match_token(TokenKind::LBrace) {
196            return None;
197        }
198
199        let mut map = std::collections::HashMap::new();
200
201        while !self.check_token(TokenKind::RBrace) && !self.is_eof() {
202            // Skip separators and formatting before the key
203            while self.check_token(TokenKind::Newline)
204                || self.check_token(TokenKind::Whitespace)
205                || self.check_token(TokenKind::Indent)
206                || self.check_token(TokenKind::Dedent)
207                || self.check_token(TokenKind::Comma)
208            {
209                self.advance();
210            }
211
212            // Check if we are at the closing brace of the map
213            if self.check_token(TokenKind::RBrace) {
214                break;
215            }
216
217            let key = if let Some(token) = self.advance() {
218                match token.kind {
219                    TokenKind::Whitespace
220                    | TokenKind::Indent
221                    | TokenKind::Dedent
222                    | TokenKind::Newline => {
223                        continue;
224                    }
225                    _ => token.lexeme.clone(),
226                }
227            } else {
228                break;
229            };
230
231            // Skip newlines and whitespace before colon
232            while self.check_token(TokenKind::Newline) || self.check_token(TokenKind::Whitespace) {
233                self.advance();
234            }
235
236            if !self.match_token(TokenKind::Colon) {
237                println!("Expected ':' after map key '{}'", key);
238                break;
239            }
240
241            // Skip separators and formatting before value
242            while self.check_token(TokenKind::Newline)
243                || self.check_token(TokenKind::Whitespace)
244                || self.check_token(TokenKind::Indent)
245                || self.check_token(TokenKind::Dedent)
246                || self.check_token(TokenKind::Comma)
247            {
248                self.advance();
249            }
250
251            let value = if let Some(token) = self.peek_clone() {
252                match token.kind {
253                    TokenKind::String => {
254                        self.advance();
255                        Value::String(token.lexeme.clone())
256                    }
257                    TokenKind::Number => {
258                        let mut number_str = token.lexeme.clone();
259                        self.advance(); // consume the first number
260
261                        if let Some(dot_token) = self.peek_clone() {
262                            if dot_token.kind == TokenKind::Dot {
263                                self.advance(); // consume the dot
264
265                                if let Some(decimal_token) = self.peek_clone() {
266                                    if decimal_token.kind == TokenKind::Number {
267                                        self.advance(); // consume the number after the dot
268                                        number_str.push('.');
269                                        number_str.push_str(&decimal_token.lexeme);
270                                    } else {
271                                        println!(
272                                            "Expected number after dot, got {:?}",
273                                            decimal_token
274                                        );
275                                        return Some(Value::Null);
276                                    }
277                                } else {
278                                    println!("Expected number after dot, but reached EOF");
279                                    return Some(Value::Null);
280                                }
281                            }
282                        }
283
284                        Value::Number(number_str.parse::<f32>().unwrap_or(0.0))
285                    }
286
287                    TokenKind::Identifier => {
288                        self.advance();
289                        Value::Identifier(token.lexeme.clone())
290                    }
291                    _ => {
292                        println!("Unexpected token in map value: {:?}", token);
293                        Value::Null
294                    }
295                }
296            } else {
297                Value::Null
298            };
299
300            map.insert(key, value);
301
302            // Optionally skip a trailing comma after the value
303            while self.check_token(TokenKind::Comma)
304                || self.check_token(TokenKind::Whitespace)
305                || self.check_token(TokenKind::Newline)
306            {
307                self.advance();
308            }
309        }
310
311        if !self.match_token(TokenKind::RBrace) {
312            println!("Expected '}}' at end of map");
313        }
314
315        Some(Value::Map(map))
316    }
317
318    // Parse an array value like [1, 2, 3] or ["a", b]
319    pub fn parse_array_value(&mut self) -> Option<Value> {
320        if !self.match_token(TokenKind::LBracket) {
321            return None;
322        }
323
324        let mut arr: Vec<Value> = Vec::new();
325
326        while !self.check_token(TokenKind::RBracket) && !self.is_eof() {
327            // Skip formatting tokens
328            while self.check_token(TokenKind::Newline)
329                || self.check_token(TokenKind::Whitespace)
330                || self.check_token(TokenKind::Indent)
331                || self.check_token(TokenKind::Dedent)
332                || self.check_token(TokenKind::Comma)
333            {
334                self.advance();
335            }
336
337            if self.check_token(TokenKind::RBracket) {
338                break;
339            }
340
341            if let Some(token) = self.peek_clone() {
342                let value = match token.kind {
343                    TokenKind::String => {
344                        self.advance();
345                        Value::String(token.lexeme.clone())
346                    }
347                    TokenKind::Number => {
348                        // Support simple decimals split as number '.' number
349                        let mut number_str = token.lexeme.clone();
350                        self.advance();
351                        if let Some(dot) = self.peek_clone() {
352                            if dot.kind == TokenKind::Dot {
353                                if let Some(next) = self.peek_nth(1).cloned() {
354                                    if next.kind == TokenKind::Number {
355                                        self.advance(); // consume dot
356                                        self.advance(); // consume next number
357                                        number_str.push('.');
358                                        number_str.push_str(&next.lexeme);
359                                    }
360                                }
361                            }
362                        }
363                        Value::Number(number_str.parse::<f32>().unwrap_or(0.0))
364                    }
365                    TokenKind::Identifier => {
366                        self.advance();
367                        Value::Identifier(token.lexeme.clone())
368                    }
369                    TokenKind::LBrace => {
370                        // Allow inline maps inside arrays
371                        if let Some(v) = self.parse_map_value() {
372                            v
373                        } else {
374                            Value::Null
375                        }
376                    }
377                    TokenKind::LBracket => {
378                        // Nested arrays
379                        if let Some(v) = self.parse_array_value() {
380                            v
381                        } else {
382                            Value::Null
383                        }
384                    }
385                    _ => {
386                        self.advance();
387                        Value::Null
388                    }
389                };
390
391                // Only push non-null (retain alignment with permissive parsing)
392                if value != Value::Null {
393                    arr.push(value);
394                }
395
396                // Optional trailing comma handled by the skipper at loop start
397            } else {
398                break;
399            }
400        }
401
402        if !self.match_token(TokenKind::RBracket) {
403            println!("Expected ']' at end of array");
404        }
405
406        Some(Value::Array(arr))
407    }
408
409    pub fn peek(&self) -> Option<&Token> {
410        self.tokens.get(self.token_index)
411    }
412
413    pub fn peek_clone(&self) -> Option<Token> {
414        self.tokens.get(self.token_index).cloned()
415    }
416
417    pub fn expect(&mut self, kind: TokenKind) -> Result<&Token, String> {
418        let tok = self.advance().ok_or("Unexpected end of input")?;
419        if tok.kind == kind {
420            Ok(tok)
421        } else {
422            Err(format!("Expected {:?}, got {:?}", kind, tok.kind))
423        }
424    }
425
426    pub fn collect_block_tokens(&mut self, base_indent: usize) -> Vec<Token> {
427        let mut tokens = Vec::new();
428
429        while let Some(tok) = self.peek() {
430            if tok.indent <= base_indent && tok.kind != TokenKind::Newline {
431                break;
432            }
433            tokens.push(self.advance().unwrap().clone());
434        }
435
436        tokens
437    }
438
439    pub fn collect_until<F>(&mut self, condition: F) -> Vec<Token>
440    where
441        F: Fn(&Token) -> bool,
442    {
443        let mut collected = Vec::new();
444        while let Some(token) = self.peek() {
445            if condition(token) {
446                break;
447            }
448            if token.kind == TokenKind::EOF {
449                break;
450            }
451            collected.push(self.advance().unwrap().clone());
452        }
453
454        collected
455    }
456
457    pub fn is_eof(&self) -> bool {
458        self.token_index >= self.tokens.len()
459    }
460
461    pub fn parse_block_until_next_else(
462        &mut self,
463        base_indent: usize,
464        global_store: &mut GlobalStore,
465    ) -> Vec<Statement> {
466        let mut block_tokens = Vec::new();
467
468        while let Some(tok) = self.peek() {
469            // Stop if we encounter an 'else' at same indent level
470            if tok.lexeme == "else" && tok.indent == base_indent {
471                break;
472            }
473            block_tokens.push(self.advance().unwrap().clone());
474        }
475
476        self.parse_block(block_tokens, global_store)
477    }
478
479    pub fn parse_condition_until_colon(&mut self) -> Option<Value> {
480        let tokens = self.collect_until(|t| t.kind == TokenKind::Colon);
481        if tokens.is_empty() {
482            return None;
483        }
484
485        let condition = tokens
486            .iter()
487            .map(|t| t.lexeme.clone())
488            .collect::<Vec<_>>()
489            .join(" ");
490
491        Some(Value::String(condition))
492    }
493
494    pub fn parse_block_until_else_or_dedent(
495        &mut self,
496        base_indent: usize,
497        global_store: &mut GlobalStore,
498    ) -> Vec<Statement> {
499        let mut tokens = Vec::new();
500
501        while let Some(tok) = self.peek() {
502            if tok.lexeme == "else" && tok.indent == base_indent {
503                break;
504            }
505            if tok.indent < base_indent && tok.kind != TokenKind::Newline {
506                break;
507            }
508            tokens.push(self.advance().unwrap().clone());
509        }
510
511        self.parse_block(tokens, global_store)
512    }
513}
devalang_core/core/parser/driver.rs

devalang_core/core/parser/
driver.rs
