devalang_core/core/parser/
driver.rs

1use devalang_types::Value;
2
3use crate::core::{
4    lexer::token::{Token, TokenKind},
5    parser::{
6        handler::{
7            arrow_call::parse_arrow_call,
8            at::parse_at_token,
9            bank::parse_bank_token,
10            condition::parse_condition_token,
11            dot::parse_dot_token,
12            identifier::{
13                emit::parse_emit_token, function::parse_function_token, on::parse_on_token,
14                parse_identifier_token,
15            },
16            loop_::parse_loop_token,
17            pattern::parse_pattern_token,
18            tempo::parse_tempo_token,
19        },
20        statement::Statement,
21    },
22    store::global::GlobalStore,
23};
24
25#[derive(Debug, Clone, PartialEq)]
26pub struct Parser {
27    pub resolve_modules: bool,
28    pub tokens: Vec<Token>,
29    pub token_index: usize,
30    pub current_module: String,
31    pub previous: Option<Token>,
32}
33
34impl Default for Parser {
35    fn default() -> Self {
36        Self::new()
37    }
38}
39
40impl Parser {
41    pub fn new() -> Self {
42        Parser {
43            resolve_modules: false,
44            tokens: Vec::new(),
45            token_index: 0,
46            current_module: String::new(),
47            previous: None,
48        }
49    }
50
51    pub fn set_current_module(&mut self, module_path: String) {
52        self.current_module = module_path;
53    }
54
55    pub fn advance(&mut self) -> Option<&Token> {
56        if self.is_eof() {
57            return None;
58        }
59
60        self.previous = self.tokens.get(self.token_index).cloned();
61        self.token_index += 1;
62
63        self.tokens.get(self.token_index - 1)
64    }
65
66    pub fn peek_is(&self, expected: &str) -> bool {
67        self.peek().is_some_and(|t| t.lexeme == expected)
68    }
69
70    pub fn peek_nth(&self, n: usize) -> Option<&Token> {
71        if self.token_index + n < self.tokens.len() {
72            self.tokens.get(self.token_index + n)
73        } else {
74            None
75        }
76    }
77
78    pub fn peek_nth_kind(&self, n: usize) -> Option<TokenKind> {
79        self.peek_nth(n).map(|t| t.kind.clone())
80    }
81
82    pub fn advance_if(&mut self, kind: TokenKind) -> bool {
83        self.match_token(kind)
84    }
85
86    pub fn match_token(&mut self, kind: TokenKind) -> bool {
87        if let Some(tok) = self.peek() {
88            if tok.kind == kind {
89                self.advance();
90                return true;
91            }
92        }
93        false
94    }
95
96    pub fn previous_clone(&self) -> Option<Token> {
97        self.previous.clone()
98    }
99
100    pub fn parse_block(
101        &self,
102        tokens: Vec<Token>,
103        global_store: &mut GlobalStore,
104    ) -> Vec<Statement> {
105        let mut inner_parser = Parser {
106            resolve_modules: self.resolve_modules,
107            tokens,
108            token_index: 0,
109            current_module: self.current_module.clone(),
110            previous: None,
111        };
112
113        inner_parser.parse_tokens(inner_parser.tokens.clone(), global_store)
114    }
115
116    pub fn parse_tokens(
117        &mut self,
118        tokens: Vec<Token>,
119        global_store: &mut GlobalStore,
120    ) -> Vec<Statement> {
121        // Filter out Whitespace tokens but keep Newline tokens because some constructs (e.g., print ...)
122        // rely on end-of-line semantics.
123        self.tokens = tokens
124            .into_iter()
125            .filter(|t| t.kind != TokenKind::Whitespace)
126            .collect();
127        self.token_index = 0;
128
129        let mut statements = Vec::new();
130
131        while !self.is_eof() {
132            let token = match self.peek() {
133                Some(t) => t.clone(),
134                None => {
135                    break;
136                }
137            };
138
139            if token.kind == TokenKind::Newline {
140                self.advance();
141                continue;
142            }
143
144            let statement = match &token.kind {
145                TokenKind::At => parse_at_token(self, global_store),
146                TokenKind::Identifier => {
147                    if let Some(next) = self.peek_nth(1).cloned() {
148                        if next.kind == TokenKind::Arrow {
149                            parse_arrow_call(self, global_store)
150                        } else {
151                            parse_identifier_token(self, global_store)
152                        }
153                    } else {
154                        parse_identifier_token(self, global_store)
155                    }
156                }
157                TokenKind::Dot => parse_dot_token(self, global_store),
158                TokenKind::Tempo => parse_tempo_token(self, global_store),
159                TokenKind::Bank => parse_bank_token(self, global_store),
160                TokenKind::Pattern => parse_pattern_token(self, global_store),
161                TokenKind::Loop => parse_loop_token(self, global_store),
162                TokenKind::If => parse_condition_token(self, global_store),
163                TokenKind::Function => parse_function_token(self, global_store),
164                TokenKind::On => parse_on_token(self, global_store),
165                TokenKind::Emit => parse_emit_token(self, token.clone(), global_store),
166
167                | TokenKind::Else // Ignore else, already handled in `parse_condition_token`
168                | TokenKind::Comment
169                | TokenKind::Equals
170                | TokenKind::Colon
171                | TokenKind::Number
172                | TokenKind::String
173                | TokenKind::LBrace
174                | TokenKind::RBrace
175                | TokenKind::Comma
176                | TokenKind::Dedent
177                | TokenKind::Indent => {
178                    self.advance();
179                    continue;
180                }
181
182                TokenKind::EOF => {
183                    break;
184                }
185
186                _ => {
187                    self.advance();
188                    Statement::unknown_with_pos(token.indent, token.line, token.column)
189                }
190            };
191
192            statements.push(statement);
193        }
194
195        statements
196    }
197
198    pub fn check_token(&self, kind: TokenKind) -> bool {
199        self.peek().is_some_and(|t| t.kind == kind)
200    }
201
202    pub fn peek_kind(&self) -> Option<TokenKind> {
203        self.peek().map(|t| t.kind.clone())
204    }
205
206    pub fn parse_map_value(&mut self) -> Option<Value> {
207        let logger = devalang_utils::logger::Logger::new();
208        use devalang_utils::logger::LogLevel;
209        if !self.match_token(TokenKind::LBrace) {
210            return None;
211        }
212
213        let mut map = std::collections::HashMap::new();
214
215        while !self.check_token(TokenKind::RBrace) && !self.is_eof() {
216            // Skip separators and formatting before the key
217            while self.check_token(TokenKind::Newline)
218                || self.check_token(TokenKind::Whitespace)
219                || self.check_token(TokenKind::Indent)
220                || self.check_token(TokenKind::Dedent)
221                || self.check_token(TokenKind::Comma)
222            {
223                self.advance();
224            }
225
226            // Check if we are at the closing brace of the map
227            if self.check_token(TokenKind::RBrace) {
228                break;
229            }
230
231            let key = if let Some(token) = self.advance() {
232                match token.kind {
233                    TokenKind::Whitespace
234                    | TokenKind::Indent
235                    | TokenKind::Dedent
236                    | TokenKind::Newline => {
237                        continue;
238                    }
239                    _ => token.lexeme.clone(),
240                }
241            } else {
242                break;
243            };
244
245            // Skip newlines and whitespace before colon
246            while self.check_token(TokenKind::Newline) || self.check_token(TokenKind::Whitespace) {
247                self.advance();
248            }
249
250            if !self.match_token(TokenKind::Colon) {
251                logger.log_message(
252                    LogLevel::Error,
253                    &format!("Expected ':' after map key '{}'", key),
254                );
255                break;
256            }
257
258            // Skip separators and formatting before value
259            while self.check_token(TokenKind::Newline)
260                || self.check_token(TokenKind::Whitespace)
261                || self.check_token(TokenKind::Indent)
262                || self.check_token(TokenKind::Dedent)
263                || self.check_token(TokenKind::Comma)
264            {
265                self.advance();
266            }
267
268            let value = if let Some(token) = self.peek_clone() {
269                match token.kind {
270                    TokenKind::String => {
271                        self.advance();
272                        Value::String(token.lexeme.clone())
273                    }
274                    TokenKind::Number => {
275                        let mut number_str = token.lexeme.clone();
276                        self.advance(); // consume the first number
277
278                        if let Some(dot_token) = self.peek_clone() {
279                            if dot_token.kind == TokenKind::Dot {
280                                self.advance(); // consume the dot
281
282                                if let Some(decimal_token) = self.peek_clone() {
283                                    if decimal_token.kind == TokenKind::Number {
284                                        self.advance(); // consume the number after the dot
285                                        number_str.push('.');
286                                        number_str.push_str(&decimal_token.lexeme);
287                                    } else {
288                                        logger.log_message(
289                                            LogLevel::Error,
290                                            &format!(
291                                                "Expected number after dot, got {:?}",
292                                                decimal_token
293                                            ),
294                                        );
295                                        return Some(Value::Null);
296                                    }
297                                } else {
298                                    logger.log_message(
299                                        LogLevel::Error,
300                                        "Expected number after dot, but reached EOF",
301                                    );
302                                    return Some(Value::Null);
303                                }
304                            }
305                        }
306
307                        Value::Number(number_str.parse::<f32>().unwrap_or(0.0))
308                    }
309
310                    TokenKind::Identifier => {
311                        // Support dotted identifiers in map values: alias.param or nested
312                        let current_line = token.line;
313                        let mut parts: Vec<String> = vec![token.lexeme.clone()];
314                        self.advance();
315                        loop {
316                            let Some(next) = self.peek_clone() else { break };
317                            if next.line != current_line {
318                                break;
319                            }
320                            if next.kind == TokenKind::Dot {
321                                // Consume '.' and the following identifier/number on same line
322                                self.advance(); // dot
323                                if let Some(id2) = self.peek_clone() {
324                                    if id2.line == current_line
325                                        && (id2.kind == TokenKind::Identifier
326                                            || id2.kind == TokenKind::Number)
327                                    {
328                                        parts.push(id2.lexeme.clone());
329                                        self.advance(); // consume part
330                                        continue;
331                                    }
332                                }
333                                break;
334                            } else {
335                                break;
336                            }
337                        }
338                        Value::Identifier(parts.join("."))
339                    }
340                    _ => {
341                        logger.log_message(
342                            LogLevel::Error,
343                            &format!("Unexpected token in map value: {:?}", token),
344                        );
345                        Value::Null
346                    }
347                }
348            } else {
349                Value::Null
350            };
351
352            map.insert(key, value);
353
354            // Optionally skip a trailing comma after the value
355            while self.check_token(TokenKind::Comma)
356                || self.check_token(TokenKind::Whitespace)
357                || self.check_token(TokenKind::Newline)
358            {
359                self.advance();
360            }
361        }
362
363        if !self.match_token(TokenKind::RBrace) {
364            logger.log_message(LogLevel::Error, "Expected '}' at end of map");
365        }
366
367        Some(Value::Map(map))
368    }
369
370    // Parse an array value like [1, 2, 3] or ["a", b]
371    pub fn parse_array_value(&mut self) -> Option<Value> {
372        let logger = devalang_utils::logger::Logger::new();
373        use devalang_utils::logger::LogLevel;
374        if !self.match_token(TokenKind::LBracket) {
375            return None;
376        }
377
378        let mut arr: Vec<Value> = Vec::new();
379
380        while !self.check_token(TokenKind::RBracket) && !self.is_eof() {
381            // Skip formatting tokens
382            while self.check_token(TokenKind::Newline)
383                || self.check_token(TokenKind::Whitespace)
384                || self.check_token(TokenKind::Indent)
385                || self.check_token(TokenKind::Dedent)
386                || self.check_token(TokenKind::Comma)
387            {
388                self.advance();
389            }
390
391            if self.check_token(TokenKind::RBracket) {
392                break;
393            }
394
395            if let Some(token) = self.peek_clone() {
396                let value = match token.kind {
397                    TokenKind::String => {
398                        self.advance();
399                        Value::String(token.lexeme.clone())
400                    }
401                    TokenKind::Number => {
402                        // Support simple decimals split as number '.' number
403                        let mut number_str = token.lexeme.clone();
404                        self.advance();
405                        if let Some(dot) = self.peek_clone() {
406                            if dot.kind == TokenKind::Dot {
407                                if let Some(next) = self.peek_nth(1).cloned() {
408                                    if next.kind == TokenKind::Number {
409                                        self.advance(); // consume dot
410                                        self.advance(); // consume next number
411                                        number_str.push('.');
412                                        number_str.push_str(&next.lexeme);
413                                    }
414                                }
415                            }
416                        }
417                        Value::Number(number_str.parse::<f32>().unwrap_or(0.0))
418                    }
419                    TokenKind::Identifier => {
420                        self.advance();
421                        Value::Identifier(token.lexeme.clone())
422                    }
423                    TokenKind::LBrace => {
424                        // Allow inline maps inside arrays
425                        if let Some(v) = self.parse_map_value() {
426                            v
427                        } else {
428                            Value::Null
429                        }
430                    }
431                    TokenKind::LBracket => {
432                        // Nested arrays
433                        if let Some(v) = self.parse_array_value() {
434                            v
435                        } else {
436                            Value::Null
437                        }
438                    }
439                    _ => {
440                        self.advance();
441                        Value::Null
442                    }
443                };
444
445                // Only push non-null (retain alignment with permissive parsing)
446                if value != Value::Null {
447                    arr.push(value);
448                }
449
450                // Optional trailing comma handled by the skipper at loop start
451            } else {
452                break;
453            }
454        }
455
456        if !self.match_token(TokenKind::RBracket) {
457            logger.log_message(LogLevel::Error, "Expected ']' at end of array");
458        }
459
460        Some(Value::Array(arr))
461    }
462
463    pub fn peek(&self) -> Option<&Token> {
464        self.tokens.get(self.token_index)
465    }
466
467    pub fn peek_clone(&self) -> Option<Token> {
468        self.tokens.get(self.token_index).cloned()
469    }
470
471    pub fn expect(&mut self, kind: TokenKind) -> Result<&Token, String> {
472        let tok = self.advance().ok_or("Unexpected end of input")?;
473        if tok.kind == kind {
474            Ok(tok)
475        } else {
476            Err(format!("Expected {:?}, got {:?}", kind, tok.kind))
477        }
478    }
479
480    pub fn collect_block_tokens(&mut self, base_indent: usize) -> Vec<Token> {
481        let mut tokens = Vec::new();
482
483        while let Some(tok) = self.peek() {
484            if tok.indent <= base_indent && tok.kind != TokenKind::Newline {
485                break;
486            }
487            if let Some(t) = self.advance() {
488                tokens.push(t.clone());
489            } else {
490                // Unexpected EOF while collecting block tokens: stop collecting
491                break;
492            }
493        }
494
495        tokens
496    }
497
498    pub fn collect_until<F>(&mut self, condition: F) -> Vec<Token>
499    where
500        F: Fn(&Token) -> bool,
501    {
502        let mut collected = Vec::new();
503        while let Some(token) = self.peek() {
504            if condition(token) {
505                break;
506            }
507            if token.kind == TokenKind::EOF {
508                break;
509            }
510            if let Some(t) = self.advance() {
511                collected.push(t.clone());
512            } else {
513                break;
514            }
515        }
516
517        collected
518    }
519
520    pub fn is_eof(&self) -> bool {
521        self.token_index >= self.tokens.len()
522    }
523
524    pub fn parse_block_until_next_else(
525        &mut self,
526        base_indent: usize,
527        global_store: &mut GlobalStore,
528    ) -> Vec<Statement> {
529        let mut block_tokens = Vec::new();
530
531        while let Some(tok) = self.peek() {
532            // Stop if we encounter an 'else' at same indent level
533            if tok.lexeme == "else" && tok.indent == base_indent {
534                break;
535            }
536            if let Some(t) = self.advance() {
537                block_tokens.push(t.clone());
538            } else {
539                break;
540            }
541        }
542
543        self.parse_block(block_tokens, global_store)
544    }
545
546    pub fn parse_condition_until_colon(&mut self) -> Option<Value> {
547        let tokens = self.collect_until(|t| t.kind == TokenKind::Colon);
548        if tokens.is_empty() {
549            return None;
550        }
551
552        let condition = tokens
553            .iter()
554            .map(|t| t.lexeme.clone())
555            .collect::<Vec<_>>()
556            .join(" ");
557
558        Some(Value::String(condition))
559    }
560
561    pub fn parse_block_until_else_or_dedent(
562        &mut self,
563        base_indent: usize,
564        global_store: &mut GlobalStore,
565    ) -> Vec<Statement> {
566        let mut tokens = Vec::new();
567
568        while let Some(tok) = self.peek() {
569            if tok.lexeme == "else" && tok.indent == base_indent {
570                break;
571            }
572            if tok.indent < base_indent && tok.kind != TokenKind::Newline {
573                break;
574            }
575            if let Some(t) = self.advance() {
576                tokens.push(t.clone());
577            } else {
578                break;
579            }
580        }
581
582        self.parse_block(tokens, global_store)
583    }
584}
devalang_core/core/parser/driver.rs

devalang_core/core/parser/
driver.rs
