devalang_core/core/parser/
driver.rs

1use devalang_types::Value;
2
3use crate::core::{
4    lexer::token::{Token, TokenKind},
5    parser::{
6        handler::{
7            arrow_call::parse_arrow_call,
8            at::parse_at_token,
9            bank::parse_bank_token,
10            condition::parse_condition_token,
11            dot::parse_dot_token,
12            identifier::{
13                emit::parse_emit_token, function::parse_function_token, on::parse_on_token,
14                parse_identifier_token,
15            },
16            loop_::parse_loop_token,
17            tempo::parse_tempo_token,
18        },
19        statement::Statement,
20    },
21    store::global::GlobalStore,
22};
23
24#[derive(Debug, Clone, PartialEq)]
25pub struct Parser {
26    pub resolve_modules: bool,
27    pub tokens: Vec<Token>,
28    pub token_index: usize,
29    pub current_module: String,
30    pub previous: Option<Token>,
31}
32
33impl Default for Parser {
34    fn default() -> Self {
35        Self::new()
36    }
37}
38
39impl Parser {
40    pub fn new() -> Self {
41        Parser {
42            resolve_modules: false,
43            tokens: Vec::new(),
44            token_index: 0,
45            current_module: String::new(),
46            previous: None,
47        }
48    }
49
50    pub fn set_current_module(&mut self, module_path: String) {
51        self.current_module = module_path;
52    }
53
54    pub fn advance(&mut self) -> Option<&Token> {
55        if self.is_eof() {
56            return None;
57        }
58
59        self.previous = self.tokens.get(self.token_index).cloned();
60        self.token_index += 1;
61
62        self.tokens.get(self.token_index - 1)
63    }
64
65    pub fn peek_is(&self, expected: &str) -> bool {
66        self.peek().is_some_and(|t| t.lexeme == expected)
67    }
68
69    pub fn peek_nth(&self, n: usize) -> Option<&Token> {
70        if self.token_index + n < self.tokens.len() {
71            self.tokens.get(self.token_index + n)
72        } else {
73            None
74        }
75    }
76
77    pub fn peek_nth_kind(&self, n: usize) -> Option<TokenKind> {
78        self.peek_nth(n).map(|t| t.kind.clone())
79    }
80
81    pub fn advance_if(&mut self, kind: TokenKind) -> bool {
82        self.match_token(kind)
83    }
84
85    pub fn match_token(&mut self, kind: TokenKind) -> bool {
86        if let Some(tok) = self.peek() {
87            if tok.kind == kind {
88                self.advance();
89                return true;
90            }
91        }
92        false
93    }
94
95    pub fn previous_clone(&self) -> Option<Token> {
96        self.previous.clone()
97    }
98
99    pub fn parse_block(
100        &self,
101        tokens: Vec<Token>,
102        global_store: &mut GlobalStore,
103    ) -> Vec<Statement> {
104        let mut inner_parser = Parser {
105            resolve_modules: self.resolve_modules,
106            tokens,
107            token_index: 0,
108            current_module: self.current_module.clone(),
109            previous: None,
110        };
111
112        inner_parser.parse_tokens(inner_parser.tokens.clone(), global_store)
113    }
114
115    pub fn parse_tokens(
116        &mut self,
117        tokens: Vec<Token>,
118        global_store: &mut GlobalStore,
119    ) -> Vec<Statement> {
120        // Filter out Whitespace tokens but keep Newline tokens because some constructs (e.g., print ...)
121        // rely on end-of-line semantics.
122        self.tokens = tokens
123            .into_iter()
124            .filter(|t| t.kind != TokenKind::Whitespace)
125            .collect();
126        self.token_index = 0;
127
128        let mut statements = Vec::new();
129
130        while !self.is_eof() {
131            let token = match self.peek() {
132                Some(t) => t.clone(),
133                None => {
134                    break;
135                }
136            };
137
138            if token.kind == TokenKind::Newline {
139                self.advance();
140                continue;
141            }
142
143            let statement = match &token.kind {
144                TokenKind::At => parse_at_token(self, global_store),
145                TokenKind::Identifier => {
146                    if let Some(next) = self.peek_nth(1).cloned() {
147                        if next.kind == TokenKind::Arrow {
148                            parse_arrow_call(self, global_store)
149                        } else {
150                            parse_identifier_token(self, global_store)
151                        }
152                    } else {
153                        parse_identifier_token(self, global_store)
154                    }
155                }
156                TokenKind::Dot => parse_dot_token(self, global_store),
157                TokenKind::Tempo => parse_tempo_token(self, global_store),
158                TokenKind::Bank => parse_bank_token(self, global_store),
159                TokenKind::Loop => parse_loop_token(self, global_store),
160                TokenKind::If => parse_condition_token(self, global_store),
161                TokenKind::Function => parse_function_token(self, global_store),
162                TokenKind::On => parse_on_token(self, global_store),
163                TokenKind::Emit => parse_emit_token(self, token.clone(), global_store),
164
165                | TokenKind::Else // Ignore else, already handled in `parse_condition_token`
166                | TokenKind::Comment
167                | TokenKind::Equals
168                | TokenKind::Colon
169                | TokenKind::Number
170                | TokenKind::String
171                | TokenKind::LBrace
172                | TokenKind::RBrace
173                | TokenKind::Comma
174                | TokenKind::Dedent
175                | TokenKind::Indent => {
176                    self.advance();
177                    continue;
178                }
179
180                TokenKind::EOF => {
181                    break;
182                }
183
184                _ => {
185                    self.advance();
186                    Statement::unknown_with_pos(token.indent, token.line, token.column)
187                }
188            };
189
190            statements.push(statement);
191        }
192
193        statements
194    }
195
196    pub fn check_token(&self, kind: TokenKind) -> bool {
197        self.peek().is_some_and(|t| t.kind == kind)
198    }
199
200    pub fn peek_kind(&self) -> Option<TokenKind> {
201        self.peek().map(|t| t.kind.clone())
202    }
203
204    pub fn parse_map_value(&mut self) -> Option<Value> {
205        if !self.match_token(TokenKind::LBrace) {
206            return None;
207        }
208
209        let mut map = std::collections::HashMap::new();
210
211        while !self.check_token(TokenKind::RBrace) && !self.is_eof() {
212            // Skip separators and formatting before the key
213            while self.check_token(TokenKind::Newline)
214                || self.check_token(TokenKind::Whitespace)
215                || self.check_token(TokenKind::Indent)
216                || self.check_token(TokenKind::Dedent)
217                || self.check_token(TokenKind::Comma)
218            {
219                self.advance();
220            }
221
222            // Check if we are at the closing brace of the map
223            if self.check_token(TokenKind::RBrace) {
224                break;
225            }
226
227            let key = if let Some(token) = self.advance() {
228                match token.kind {
229                    TokenKind::Whitespace
230                    | TokenKind::Indent
231                    | TokenKind::Dedent
232                    | TokenKind::Newline => {
233                        continue;
234                    }
235                    _ => token.lexeme.clone(),
236                }
237            } else {
238                break;
239            };
240
241            // Skip newlines and whitespace before colon
242            while self.check_token(TokenKind::Newline) || self.check_token(TokenKind::Whitespace) {
243                self.advance();
244            }
245
246            if !self.match_token(TokenKind::Colon) {
247                println!("Expected ':' after map key '{}'", key);
248                break;
249            }
250
251            // Skip separators and formatting before value
252            while self.check_token(TokenKind::Newline)
253                || self.check_token(TokenKind::Whitespace)
254                || self.check_token(TokenKind::Indent)
255                || self.check_token(TokenKind::Dedent)
256                || self.check_token(TokenKind::Comma)
257            {
258                self.advance();
259            }
260
261            let value = if let Some(token) = self.peek_clone() {
262                match token.kind {
263                    TokenKind::String => {
264                        self.advance();
265                        Value::String(token.lexeme.clone())
266                    }
267                    TokenKind::Number => {
268                        let mut number_str = token.lexeme.clone();
269                        self.advance(); // consume the first number
270
271                        if let Some(dot_token) = self.peek_clone() {
272                            if dot_token.kind == TokenKind::Dot {
273                                self.advance(); // consume the dot
274
275                                if let Some(decimal_token) = self.peek_clone() {
276                                    if decimal_token.kind == TokenKind::Number {
277                                        self.advance(); // consume the number after the dot
278                                        number_str.push('.');
279                                        number_str.push_str(&decimal_token.lexeme);
280                                    } else {
281                                        println!(
282                                            "Expected number after dot, got {:?}",
283                                            decimal_token
284                                        );
285                                        return Some(Value::Null);
286                                    }
287                                } else {
288                                    println!("Expected number after dot, but reached EOF");
289                                    return Some(Value::Null);
290                                }
291                            }
292                        }
293
294                        Value::Number(number_str.parse::<f32>().unwrap_or(0.0))
295                    }
296
297                    TokenKind::Identifier => {
298                        // Support dotted identifiers in map values: alias.param or nested
299                        let current_line = token.line;
300                        let mut parts: Vec<String> = vec![token.lexeme.clone()];
301                        self.advance();
302                        loop {
303                            let Some(next) = self.peek_clone() else { break };
304                            if next.line != current_line {
305                                break;
306                            }
307                            if next.kind == TokenKind::Dot {
308                                // Consume '.' and the following identifier/number on same line
309                                self.advance(); // dot
310                                if let Some(id2) = self.peek_clone() {
311                                    if id2.line == current_line
312                                        && (id2.kind == TokenKind::Identifier
313                                            || id2.kind == TokenKind::Number)
314                                    {
315                                        parts.push(id2.lexeme.clone());
316                                        self.advance(); // consume part
317                                        continue;
318                                    }
319                                }
320                                break;
321                            } else {
322                                break;
323                            }
324                        }
325                        Value::Identifier(parts.join("."))
326                    }
327                    _ => {
328                        println!("Unexpected token in map value: {:?}", token);
329                        Value::Null
330                    }
331                }
332            } else {
333                Value::Null
334            };
335
336            map.insert(key, value);
337
338            // Optionally skip a trailing comma after the value
339            while self.check_token(TokenKind::Comma)
340                || self.check_token(TokenKind::Whitespace)
341                || self.check_token(TokenKind::Newline)
342            {
343                self.advance();
344            }
345        }
346
347        if !self.match_token(TokenKind::RBrace) {
348            println!("Expected '}}' at end of map");
349        }
350
351        Some(Value::Map(map))
352    }
353
354    // Parse an array value like [1, 2, 3] or ["a", b]
355    pub fn parse_array_value(&mut self) -> Option<Value> {
356        if !self.match_token(TokenKind::LBracket) {
357            return None;
358        }
359
360        let mut arr: Vec<Value> = Vec::new();
361
362        while !self.check_token(TokenKind::RBracket) && !self.is_eof() {
363            // Skip formatting tokens
364            while self.check_token(TokenKind::Newline)
365                || self.check_token(TokenKind::Whitespace)
366                || self.check_token(TokenKind::Indent)
367                || self.check_token(TokenKind::Dedent)
368                || self.check_token(TokenKind::Comma)
369            {
370                self.advance();
371            }
372
373            if self.check_token(TokenKind::RBracket) {
374                break;
375            }
376
377            if let Some(token) = self.peek_clone() {
378                let value = match token.kind {
379                    TokenKind::String => {
380                        self.advance();
381                        Value::String(token.lexeme.clone())
382                    }
383                    TokenKind::Number => {
384                        // Support simple decimals split as number '.' number
385                        let mut number_str = token.lexeme.clone();
386                        self.advance();
387                        if let Some(dot) = self.peek_clone() {
388                            if dot.kind == TokenKind::Dot {
389                                if let Some(next) = self.peek_nth(1).cloned() {
390                                    if next.kind == TokenKind::Number {
391                                        self.advance(); // consume dot
392                                        self.advance(); // consume next number
393                                        number_str.push('.');
394                                        number_str.push_str(&next.lexeme);
395                                    }
396                                }
397                            }
398                        }
399                        Value::Number(number_str.parse::<f32>().unwrap_or(0.0))
400                    }
401                    TokenKind::Identifier => {
402                        self.advance();
403                        Value::Identifier(token.lexeme.clone())
404                    }
405                    TokenKind::LBrace => {
406                        // Allow inline maps inside arrays
407                        if let Some(v) = self.parse_map_value() {
408                            v
409                        } else {
410                            Value::Null
411                        }
412                    }
413                    TokenKind::LBracket => {
414                        // Nested arrays
415                        if let Some(v) = self.parse_array_value() {
416                            v
417                        } else {
418                            Value::Null
419                        }
420                    }
421                    _ => {
422                        self.advance();
423                        Value::Null
424                    }
425                };
426
427                // Only push non-null (retain alignment with permissive parsing)
428                if value != Value::Null {
429                    arr.push(value);
430                }
431
432                // Optional trailing comma handled by the skipper at loop start
433            } else {
434                break;
435            }
436        }
437
438        if !self.match_token(TokenKind::RBracket) {
439            println!("Expected ']' at end of array");
440        }
441
442        Some(Value::Array(arr))
443    }
444
445    pub fn peek(&self) -> Option<&Token> {
446        self.tokens.get(self.token_index)
447    }
448
449    pub fn peek_clone(&self) -> Option<Token> {
450        self.tokens.get(self.token_index).cloned()
451    }
452
453    pub fn expect(&mut self, kind: TokenKind) -> Result<&Token, String> {
454        let tok = self.advance().ok_or("Unexpected end of input")?;
455        if tok.kind == kind {
456            Ok(tok)
457        } else {
458            Err(format!("Expected {:?}, got {:?}", kind, tok.kind))
459        }
460    }
461
462    pub fn collect_block_tokens(&mut self, base_indent: usize) -> Vec<Token> {
463        let mut tokens = Vec::new();
464
465        while let Some(tok) = self.peek() {
466            if tok.indent <= base_indent && tok.kind != TokenKind::Newline {
467                break;
468            }
469            if let Some(t) = self.advance() {
470                tokens.push(t.clone());
471            } else {
472                // Unexpected EOF while collecting block tokens: stop collecting
473                break;
474            }
475        }
476
477        tokens
478    }
479
480    pub fn collect_until<F>(&mut self, condition: F) -> Vec<Token>
481    where
482        F: Fn(&Token) -> bool,
483    {
484        let mut collected = Vec::new();
485        while let Some(token) = self.peek() {
486            if condition(token) {
487                break;
488            }
489            if token.kind == TokenKind::EOF {
490                break;
491            }
492            if let Some(t) = self.advance() {
493                collected.push(t.clone());
494            } else {
495                break;
496            }
497        }
498
499        collected
500    }
501
502    pub fn is_eof(&self) -> bool {
503        self.token_index >= self.tokens.len()
504    }
505
506    pub fn parse_block_until_next_else(
507        &mut self,
508        base_indent: usize,
509        global_store: &mut GlobalStore,
510    ) -> Vec<Statement> {
511        let mut block_tokens = Vec::new();
512
513        while let Some(tok) = self.peek() {
514            // Stop if we encounter an 'else' at same indent level
515            if tok.lexeme == "else" && tok.indent == base_indent {
516                break;
517            }
518            if let Some(t) = self.advance() {
519                block_tokens.push(t.clone());
520            } else {
521                break;
522            }
523        }
524
525        self.parse_block(block_tokens, global_store)
526    }
527
528    pub fn parse_condition_until_colon(&mut self) -> Option<Value> {
529        let tokens = self.collect_until(|t| t.kind == TokenKind::Colon);
530        if tokens.is_empty() {
531            return None;
532        }
533
534        let condition = tokens
535            .iter()
536            .map(|t| t.lexeme.clone())
537            .collect::<Vec<_>>()
538            .join(" ");
539
540        Some(Value::String(condition))
541    }
542
543    pub fn parse_block_until_else_or_dedent(
544        &mut self,
545        base_indent: usize,
546        global_store: &mut GlobalStore,
547    ) -> Vec<Statement> {
548        let mut tokens = Vec::new();
549
550        while let Some(tok) = self.peek() {
551            if tok.lexeme == "else" && tok.indent == base_indent {
552                break;
553            }
554            if tok.indent < base_indent && tok.kind != TokenKind::Newline {
555                break;
556            }
557            if let Some(t) = self.advance() {
558                tokens.push(t.clone());
559            } else {
560                break;
561            }
562        }
563
564        self.parse_block(tokens, global_store)
565    }
566}
devalang_core/core/parser/driver.rs

devalang_core/core/parser/
driver.rs
