devalang_core/core/parser/
driver.rs

1use devalang_types::Value;
2
3use crate::core::{
4    lexer::token::{Token, TokenKind},
5    parser::{
6        handler::{
7            arrow_call::parse_arrow_call,
8            at::parse_at_token,
9            bank::parse_bank_token,
10            condition::parse_condition_token,
11            dot::parse_dot_token,
12            pattern::parse_pattern_token,
13            identifier::{
14                emit::parse_emit_token, function::parse_function_token, on::parse_on_token,
15                parse_identifier_token,
16            },
17            loop_::parse_loop_token,
18            tempo::parse_tempo_token,
19        },
20        statement::Statement,
21    },
22    store::global::GlobalStore,
23};
24
25#[derive(Debug, Clone, PartialEq)]
26pub struct Parser {
27    pub resolve_modules: bool,
28    pub tokens: Vec<Token>,
29    pub token_index: usize,
30    pub current_module: String,
31    pub previous: Option<Token>,
32}
33
34impl Default for Parser {
35    fn default() -> Self {
36        Self::new()
37    }
38}
39
40impl Parser {
41    pub fn new() -> Self {
42        Parser {
43            resolve_modules: false,
44            tokens: Vec::new(),
45            token_index: 0,
46            current_module: String::new(),
47            previous: None,
48        }
49    }
50
51    pub fn set_current_module(&mut self, module_path: String) {
52        self.current_module = module_path;
53    }
54
55    pub fn advance(&mut self) -> Option<&Token> {
56        if self.is_eof() {
57            return None;
58        }
59
60        self.previous = self.tokens.get(self.token_index).cloned();
61        self.token_index += 1;
62
63        self.tokens.get(self.token_index - 1)
64    }
65
66    pub fn peek_is(&self, expected: &str) -> bool {
67        self.peek().is_some_and(|t| t.lexeme == expected)
68    }
69
70    pub fn peek_nth(&self, n: usize) -> Option<&Token> {
71        if self.token_index + n < self.tokens.len() {
72            self.tokens.get(self.token_index + n)
73        } else {
74            None
75        }
76    }
77
78    pub fn peek_nth_kind(&self, n: usize) -> Option<TokenKind> {
79        self.peek_nth(n).map(|t| t.kind.clone())
80    }
81
82    pub fn advance_if(&mut self, kind: TokenKind) -> bool {
83        self.match_token(kind)
84    }
85
86    pub fn match_token(&mut self, kind: TokenKind) -> bool {
87        if let Some(tok) = self.peek() {
88            if tok.kind == kind {
89                self.advance();
90                return true;
91            }
92        }
93        false
94    }
95
96    pub fn previous_clone(&self) -> Option<Token> {
97        self.previous.clone()
98    }
99
100    pub fn parse_block(
101        &self,
102        tokens: Vec<Token>,
103        global_store: &mut GlobalStore,
104    ) -> Vec<Statement> {
105        let mut inner_parser = Parser {
106            resolve_modules: self.resolve_modules,
107            tokens,
108            token_index: 0,
109            current_module: self.current_module.clone(),
110            previous: None,
111        };
112
113        inner_parser.parse_tokens(inner_parser.tokens.clone(), global_store)
114    }
115
116    pub fn parse_tokens(
117        &mut self,
118        tokens: Vec<Token>,
119        global_store: &mut GlobalStore,
120    ) -> Vec<Statement> {
121        // Filter out Whitespace tokens but keep Newline tokens because some constructs (e.g., print ...)
122        // rely on end-of-line semantics.
123        self.tokens = tokens
124            .into_iter()
125            .filter(|t| t.kind != TokenKind::Whitespace)
126            .collect();
127        self.token_index = 0;
128
129        let mut statements = Vec::new();
130
131        while !self.is_eof() {
132            let token = match self.peek() {
133                Some(t) => t.clone(),
134                None => {
135                    break;
136                }
137            };
138
139            if token.kind == TokenKind::Newline {
140                self.advance();
141                continue;
142            }
143
144            let statement = match &token.kind {
145                TokenKind::At => parse_at_token(self, global_store),
146                TokenKind::Identifier => {
147                    if let Some(next) = self.peek_nth(1).cloned() {
148                        if next.kind == TokenKind::Arrow {
149                            parse_arrow_call(self, global_store)
150                        } else {
151                            parse_identifier_token(self, global_store)
152                        }
153                    } else {
154                        parse_identifier_token(self, global_store)
155                    }
156                }
157                TokenKind::Dot => parse_dot_token(self, global_store),
158                TokenKind::Tempo => parse_tempo_token(self, global_store),
159                TokenKind::Bank => parse_bank_token(self, global_store),
160                TokenKind::Pattern => parse_pattern_token(self, global_store),
161                TokenKind::Loop => parse_loop_token(self, global_store),
162                TokenKind::If => parse_condition_token(self, global_store),
163                TokenKind::Function => parse_function_token(self, global_store),
164                TokenKind::On => parse_on_token(self, global_store),
165                TokenKind::Emit => parse_emit_token(self, token.clone(), global_store),
166
167                | TokenKind::Else // Ignore else, already handled in `parse_condition_token`
168                | TokenKind::Comment
169                | TokenKind::Equals
170                | TokenKind::Colon
171                | TokenKind::Number
172                | TokenKind::String
173                | TokenKind::LBrace
174                | TokenKind::RBrace
175                | TokenKind::Comma
176                | TokenKind::Dedent
177                | TokenKind::Indent => {
178                    self.advance();
179                    continue;
180                }
181
182                TokenKind::EOF => {
183                    break;
184                }
185
186                _ => {
187                    self.advance();
188                    Statement::unknown_with_pos(token.indent, token.line, token.column)
189                }
190            };
191
192            statements.push(statement);
193        }
194
195        statements
196    }
197
198    pub fn check_token(&self, kind: TokenKind) -> bool {
199        self.peek().is_some_and(|t| t.kind == kind)
200    }
201
202    pub fn peek_kind(&self) -> Option<TokenKind> {
203        self.peek().map(|t| t.kind.clone())
204    }
205
206    pub fn parse_map_value(&mut self) -> Option<Value> {
207    let logger = devalang_utils::logger::Logger::new();
208    use devalang_utils::logger::LogLevel;
209        if !self.match_token(TokenKind::LBrace) {
210            return None;
211        }
212
213        let mut map = std::collections::HashMap::new();
214
215        while !self.check_token(TokenKind::RBrace) && !self.is_eof() {
216            // Skip separators and formatting before the key
217            while self.check_token(TokenKind::Newline)
218                || self.check_token(TokenKind::Whitespace)
219                || self.check_token(TokenKind::Indent)
220                || self.check_token(TokenKind::Dedent)
221                || self.check_token(TokenKind::Comma)
222            {
223                self.advance();
224            }
225
226            // Check if we are at the closing brace of the map
227            if self.check_token(TokenKind::RBrace) {
228                break;
229            }
230
231            let key = if let Some(token) = self.advance() {
232                match token.kind {
233                    TokenKind::Whitespace
234                    | TokenKind::Indent
235                    | TokenKind::Dedent
236                    | TokenKind::Newline => {
237                        continue;
238                    }
239                    _ => token.lexeme.clone(),
240                }
241            } else {
242                break;
243            };
244
245            // Skip newlines and whitespace before colon
246            while self.check_token(TokenKind::Newline) || self.check_token(TokenKind::Whitespace) {
247                self.advance();
248            }
249
250            if !self.match_token(TokenKind::Colon) {
251                logger.log_message(LogLevel::Error, &format!("Expected ':' after map key '{}'", key));
252                break;
253            }
254
255            // Skip separators and formatting before value
256            while self.check_token(TokenKind::Newline)
257                || self.check_token(TokenKind::Whitespace)
258                || self.check_token(TokenKind::Indent)
259                || self.check_token(TokenKind::Dedent)
260                || self.check_token(TokenKind::Comma)
261            {
262                self.advance();
263            }
264
265            let value = if let Some(token) = self.peek_clone() {
266                match token.kind {
267                    TokenKind::String => {
268                        self.advance();
269                        Value::String(token.lexeme.clone())
270                    }
271                    TokenKind::Number => {
272                        let mut number_str = token.lexeme.clone();
273                        self.advance(); // consume the first number
274
275                        if let Some(dot_token) = self.peek_clone() {
276                            if dot_token.kind == TokenKind::Dot {
277                                self.advance(); // consume the dot
278
279                                if let Some(decimal_token) = self.peek_clone() {
280                                    if decimal_token.kind == TokenKind::Number {
281                                        self.advance(); // consume the number after the dot
282                                        number_str.push('.');
283                                        number_str.push_str(&decimal_token.lexeme);
284                                    } else {
285                                        logger.log_message(LogLevel::Error, &format!("Expected number after dot, got {:?}", decimal_token));
286                                        return Some(Value::Null);
287                                    }
288                                } else {
289                                    logger.log_message(LogLevel::Error, "Expected number after dot, but reached EOF");
290                                    return Some(Value::Null);
291                                }
292                            }
293                        }
294
295                        Value::Number(number_str.parse::<f32>().unwrap_or(0.0))
296                    }
297
298                    TokenKind::Identifier => {
299                        // Support dotted identifiers in map values: alias.param or nested
300                        let current_line = token.line;
301                        let mut parts: Vec<String> = vec![token.lexeme.clone()];
302                        self.advance();
303                        loop {
304                            let Some(next) = self.peek_clone() else { break };
305                            if next.line != current_line {
306                                break;
307                            }
308                            if next.kind == TokenKind::Dot {
309                                // Consume '.' and the following identifier/number on same line
310                                self.advance(); // dot
311                                if let Some(id2) = self.peek_clone() {
312                                    if id2.line == current_line
313                                        && (id2.kind == TokenKind::Identifier
314                                            || id2.kind == TokenKind::Number)
315                                    {
316                                        parts.push(id2.lexeme.clone());
317                                        self.advance(); // consume part
318                                        continue;
319                                    }
320                                }
321                                break;
322                            } else {
323                                break;
324                            }
325                        }
326                        Value::Identifier(parts.join("."))
327                    }
328                    _ => {
329                        logger.log_message(LogLevel::Error, &format!("Unexpected token in map value: {:?}", token));
330                        Value::Null
331                    }
332                }
333            } else {
334                Value::Null
335            };
336
337            map.insert(key, value);
338
339            // Optionally skip a trailing comma after the value
340            while self.check_token(TokenKind::Comma)
341                || self.check_token(TokenKind::Whitespace)
342                || self.check_token(TokenKind::Newline)
343            {
344                self.advance();
345            }
346        }
347
348        if !self.match_token(TokenKind::RBrace) {
349            logger.log_message(LogLevel::Error, "Expected '}' at end of map");
350        }
351
352        Some(Value::Map(map))
353    }
354
355    // Parse an array value like [1, 2, 3] or ["a", b]
356    pub fn parse_array_value(&mut self) -> Option<Value> {
357    let logger = devalang_utils::logger::Logger::new();
358    use devalang_utils::logger::LogLevel;
359        if !self.match_token(TokenKind::LBracket) {
360            return None;
361        }
362
363        let mut arr: Vec<Value> = Vec::new();
364
365        while !self.check_token(TokenKind::RBracket) && !self.is_eof() {
366            // Skip formatting tokens
367            while self.check_token(TokenKind::Newline)
368                || self.check_token(TokenKind::Whitespace)
369                || self.check_token(TokenKind::Indent)
370                || self.check_token(TokenKind::Dedent)
371                || self.check_token(TokenKind::Comma)
372            {
373                self.advance();
374            }
375
376            if self.check_token(TokenKind::RBracket) {
377                break;
378            }
379
380            if let Some(token) = self.peek_clone() {
381                let value = match token.kind {
382                    TokenKind::String => {
383                        self.advance();
384                        Value::String(token.lexeme.clone())
385                    }
386                    TokenKind::Number => {
387                        // Support simple decimals split as number '.' number
388                        let mut number_str = token.lexeme.clone();
389                        self.advance();
390                        if let Some(dot) = self.peek_clone() {
391                            if dot.kind == TokenKind::Dot {
392                                if let Some(next) = self.peek_nth(1).cloned() {
393                                    if next.kind == TokenKind::Number {
394                                        self.advance(); // consume dot
395                                        self.advance(); // consume next number
396                                        number_str.push('.');
397                                        number_str.push_str(&next.lexeme);
398                                    }
399                                }
400                            }
401                        }
402                        Value::Number(number_str.parse::<f32>().unwrap_or(0.0))
403                    }
404                    TokenKind::Identifier => {
405                        self.advance();
406                        Value::Identifier(token.lexeme.clone())
407                    }
408                    TokenKind::LBrace => {
409                        // Allow inline maps inside arrays
410                        if let Some(v) = self.parse_map_value() {
411                            v
412                        } else {
413                            Value::Null
414                        }
415                    }
416                    TokenKind::LBracket => {
417                        // Nested arrays
418                        if let Some(v) = self.parse_array_value() {
419                            v
420                        } else {
421                            Value::Null
422                        }
423                    }
424                    _ => {
425                        self.advance();
426                        Value::Null
427                    }
428                };
429
430                // Only push non-null (retain alignment with permissive parsing)
431                if value != Value::Null {
432                    arr.push(value);
433                }
434
435                // Optional trailing comma handled by the skipper at loop start
436            } else {
437                break;
438            }
439        }
440
441        if !self.match_token(TokenKind::RBracket) {
442            logger.log_message(LogLevel::Error, "Expected ']' at end of array");
443        }
444
445        Some(Value::Array(arr))
446    }
447
448    pub fn peek(&self) -> Option<&Token> {
449        self.tokens.get(self.token_index)
450    }
451
452    pub fn peek_clone(&self) -> Option<Token> {
453        self.tokens.get(self.token_index).cloned()
454    }
455
456    pub fn expect(&mut self, kind: TokenKind) -> Result<&Token, String> {
457        let tok = self.advance().ok_or("Unexpected end of input")?;
458        if tok.kind == kind {
459            Ok(tok)
460        } else {
461            Err(format!("Expected {:?}, got {:?}", kind, tok.kind))
462        }
463    }
464
465    pub fn collect_block_tokens(&mut self, base_indent: usize) -> Vec<Token> {
466        let mut tokens = Vec::new();
467
468        while let Some(tok) = self.peek() {
469            if tok.indent <= base_indent && tok.kind != TokenKind::Newline {
470                break;
471            }
472            if let Some(t) = self.advance() {
473                tokens.push(t.clone());
474            } else {
475                // Unexpected EOF while collecting block tokens: stop collecting
476                break;
477            }
478        }
479
480        tokens
481    }
482
483    pub fn collect_until<F>(&mut self, condition: F) -> Vec<Token>
484    where
485        F: Fn(&Token) -> bool,
486    {
487        let mut collected = Vec::new();
488        while let Some(token) = self.peek() {
489            if condition(token) {
490                break;
491            }
492            if token.kind == TokenKind::EOF {
493                break;
494            }
495            if let Some(t) = self.advance() {
496                collected.push(t.clone());
497            } else {
498                break;
499            }
500        }
501
502        collected
503    }
504
505    pub fn is_eof(&self) -> bool {
506        self.token_index >= self.tokens.len()
507    }
508
509    pub fn parse_block_until_next_else(
510        &mut self,
511        base_indent: usize,
512        global_store: &mut GlobalStore,
513    ) -> Vec<Statement> {
514        let mut block_tokens = Vec::new();
515
516        while let Some(tok) = self.peek() {
517            // Stop if we encounter an 'else' at same indent level
518            if tok.lexeme == "else" && tok.indent == base_indent {
519                break;
520            }
521            if let Some(t) = self.advance() {
522                block_tokens.push(t.clone());
523            } else {
524                break;
525            }
526        }
527
528        self.parse_block(block_tokens, global_store)
529    }
530
531    pub fn parse_condition_until_colon(&mut self) -> Option<Value> {
532        let tokens = self.collect_until(|t| t.kind == TokenKind::Colon);
533        if tokens.is_empty() {
534            return None;
535        }
536
537        let condition = tokens
538            .iter()
539            .map(|t| t.lexeme.clone())
540            .collect::<Vec<_>>()
541            .join(" ");
542
543        Some(Value::String(condition))
544    }
545
546    pub fn parse_block_until_else_or_dedent(
547        &mut self,
548        base_indent: usize,
549        global_store: &mut GlobalStore,
550    ) -> Vec<Statement> {
551        let mut tokens = Vec::new();
552
553        while let Some(tok) = self.peek() {
554            if tok.lexeme == "else" && tok.indent == base_indent {
555                break;
556            }
557            if tok.indent < base_indent && tok.kind != TokenKind::Newline {
558                break;
559            }
560            if let Some(t) = self.advance() {
561                tokens.push(t.clone());
562            } else {
563                break;
564            }
565        }
566
567        self.parse_block(tokens, global_store)
568    }
569}
devalang_core/core/parser/driver.rs

devalang_core/core/parser/
driver.rs
