somni_parser/
parser.rs

1//! Grammar parser.
2//!
3//! This module parses the following grammar (minus comments, which are ignored):
4//!
5//! ```text
6//! program -> item* EOF;
7//! item -> function | global | extern_fn; // TODO types.
8//!
9//! extern_fn -> 'extern' 'fn' identifier '(' function_argument ( ',' function_argument )* ','? ')' return_decl? ;
10//! global -> 'var' identifier ':' type '=' static_initializer ';' ;
11//!
12//! static_initializer -> expression ; // The language does not currently support function calls.
13//!
14//! function -> 'fn' identifier '(' function_argument ( ',' function_argument )* ','? ')' return_decl? body ;
15//! function_argument -> identifier ':' '&'? type ;
16//! return_decl -> '->' type ;
17//! type -> identifier ;
18//!
19//! body -> '{' statement* '}' ;
20//! statement -> 'var' identifier (':' type)? '=' expression ';'
21//!            | 'return' expression? ';'
22//!            | 'break' expression? ';'
23//!            | 'continue' expression? ';'
24//!            | 'if' expression body ( 'else' body )?
25//!            | 'loop' body
26//!            | 'while' expression body
27//!            | expression ';'
28//!
29//! expression -> binary1 ;
30//! binary0 -> binary1 ( '=' binary1 )? ;
31//! binary1 -> binary2 ( '||' binary2 )* ;
32//! binary2 -> binary3 ( '&&' binary3 )* ;
33//! binary3 -> binary4 ( ( '<' | '<=' | '>' | '>=' | '==' | '!=' ) binary4 )* ;
34//! binary4 -> binary5 ( '|' binary5 )* ;
35//! binary5 -> binary6 ( '^' binary6 )* ;
36//! binary6 -> binary7 ( '&' binary7 )* ;
37//! binary7 -> binary8 ( ( '<<' | '>>' ) binary8 )* ;
38//! binary8 -> binary9 ( ( '+' | '-' ) binary9 )* ;
39//! binary9 -> unary ( ( '*' | '/' ) unary )* ;
40//! unary -> ('!' | '-' | '&' | '*' )* primary | call ;
41//! primary -> ( literal | identifier ( '(' call_arguments ')' )? ) | '(' expression ')' ;
42//! call_arguments -> expression ( ',' expression )* ','? ;
43//! literal -> NUMBER | STRING | 'true' | 'false' ;
44//! ```
45//!
46//! `NUMBER`: Non-negative integers (binary, decimal, hexadecimal) and floats.
47//! `STRING`: Double-quoted strings with escape sequences.
48
49use std::{
50    fmt::Debug,
51    num::{ParseFloatError, ParseIntError},
52};
53
54use crate::{
55    ast::{
56        Body, Break, Continue, Else, EmptyReturn, Expression, ExternalFunction, Function,
57        FunctionArgument, GlobalVariable, If, Item, Literal, LiteralValue, Loop, Program,
58        ReturnDecl, ReturnWithValue, Statement, TypeHint, VariableDefinition,
59    },
60    lexer::{Location, Token, TokenKind},
61    parser::private::Sealed,
62};
63
64mod private {
65    pub trait Sealed {}
66
67    impl Sealed for u32 {}
68    impl Sealed for u64 {}
69    impl Sealed for u128 {}
70    impl Sealed for f32 {}
71    impl Sealed for f64 {}
72}
73
74/// Parse literals into Somni integers.
75pub trait IntParser: Sized + Sealed {
76    fn parse(str: &str, radix: u32) -> Result<Self, ParseIntError>;
77}
78
79impl IntParser for u32 {
80    fn parse(str: &str, radix: u32) -> Result<Self, ParseIntError> {
81        u32::from_str_radix(str, radix)
82    }
83}
84impl IntParser for u64 {
85    fn parse(str: &str, radix: u32) -> Result<Self, ParseIntError> {
86        u64::from_str_radix(str, radix)
87    }
88}
89impl IntParser for u128 {
90    fn parse(str: &str, radix: u32) -> Result<Self, ParseIntError> {
91        u128::from_str_radix(str, radix)
92    }
93}
94
95/// Parse literals into Somni floats.
96pub trait FloatParser: Sized + Sealed {
97    fn parse(str: &str) -> Result<Self, ParseFloatError>;
98}
99
100impl FloatParser for f32 {
101    fn parse(str: &str) -> Result<Self, ParseFloatError> {
102        str.parse::<f32>()
103    }
104}
105impl FloatParser for f64 {
106    fn parse(str: &str) -> Result<Self, ParseFloatError> {
107        str.parse::<f64>()
108    }
109}
110
111/// Defines the numeric types used in the parser.
112pub trait TypeSet: Clone + Debug + Copy {
113    type Integer: IntParser + Clone + Copy + PartialEq + Debug;
114    type Float: FloatParser + Clone + Copy + PartialEq + Debug;
115}
116
117/// Use 64-bit integers and 64-bit floats (default).
118#[derive(Debug, Clone, Copy, PartialEq)]
119pub struct DefaultTypeSet;
120impl Sealed for DefaultTypeSet {}
121
122impl TypeSet for DefaultTypeSet {
123    type Integer = u64;
124    type Float = f64;
125}
126
127/// Use 32-bit integers and floats.
128#[derive(Debug, Clone, Copy, PartialEq)]
129pub struct TypeSet32;
130impl Sealed for TypeSet32 {}
131impl TypeSet for TypeSet32 {
132    type Integer = u32;
133    type Float = f32;
134}
135
136/// Use 128-bit integers and 64-bit floats.
137#[derive(Debug, Clone, Copy, PartialEq)]
138pub struct TypeSet128;
139impl Sealed for TypeSet128 {}
140impl TypeSet for TypeSet128 {
141    type Integer = u128;
142    type Float = f64;
143}
144
145#[derive(Debug, Clone, PartialEq)]
146pub struct ParserError {
147    pub location: Location,
148    pub error: String,
149}
150
151impl std::fmt::Display for ParserError {
152    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
153        write!(f, "Parse error: {}", self.error)
154    }
155}
156
157impl Program {
158    fn parse<'s>(stream: &mut TokenStream<'s>) -> Result<Self, ParserError> {
159        let mut items = Vec::new();
160
161        while !stream.end() {
162            items.push(Item::parse(stream)?);
163        }
164
165        Ok(Program { items })
166    }
167}
168
169impl Item {
170    fn parse<'s>(stream: &mut TokenStream<'s>) -> Result<Self, ParserError> {
171        if let Some(global_var) = GlobalVariable::try_parse(stream)? {
172            return Ok(Item::GlobalVariable(global_var));
173        }
174        if let Some(function) = ExternalFunction::try_parse(stream)? {
175            return Ok(Item::ExternFunction(function));
176        }
177        if let Some(function) = Function::try_parse(stream)? {
178            return Ok(Item::Function(function));
179        }
180
181        Err(stream.error("Expected global variable or function definition"))
182    }
183}
184
185impl GlobalVariable {
186    fn try_parse<'s>(stream: &mut TokenStream<'s>) -> Result<Option<Self>, ParserError> {
187        let Some(decl_token) = stream.take_match(TokenKind::Identifier, &["var"]) else {
188            return Ok(None);
189        };
190
191        let identifier = stream.expect_match(TokenKind::Identifier, &[])?;
192        let colon = stream.expect_match(TokenKind::Symbol, &[":"])?;
193        let type_token = TypeHint::parse(stream)?;
194        let equals_token = stream.expect_match(TokenKind::Symbol, &["="])?;
195        let initializer = Expression::parse(stream)?;
196        let semicolon = stream.expect_match(TokenKind::Symbol, &[";"])?;
197
198        Ok(Some(GlobalVariable {
199            decl_token,
200            identifier,
201            colon,
202            type_token,
203            equals_token,
204            initializer,
205            semicolon,
206        }))
207    }
208}
209
210impl ExternalFunction {
211    fn try_parse<'s>(stream: &mut TokenStream<'s>) -> Result<Option<Self>, ParserError> {
212        let Some(extern_fn_token) = stream.take_match(TokenKind::Identifier, &["extern"]) else {
213            return Ok(None);
214        };
215        let Some(fn_token) = stream.take_match(TokenKind::Identifier, &["fn"]) else {
216            return Ok(None);
217        };
218
219        let name = stream.expect_match(TokenKind::Identifier, &[])?;
220        let opening_paren = stream.expect_match(TokenKind::Symbol, &["("])?;
221
222        let mut arguments = Vec::new();
223        while let Some(arg_name) = stream.take_match(TokenKind::Identifier, &[]) {
224            let colon = stream.expect_match(TokenKind::Symbol, &[":"])?;
225            let reference_token = stream.take_match(TokenKind::Symbol, &["&"]);
226            let type_token = TypeHint::parse(stream)?;
227
228            arguments.push(FunctionArgument {
229                name: arg_name,
230                colon,
231                reference_token,
232                arg_type: type_token,
233            });
234
235            if stream.take_match(TokenKind::Symbol, &[","]).is_none() {
236                break;
237            }
238        }
239
240        let closing_paren = stream.expect_match(TokenKind::Symbol, &[")"])?;
241
242        let return_decl = if let Some(return_token) = stream.take_match(TokenKind::Symbol, &["->"])
243        {
244            Some(ReturnDecl {
245                return_token,
246                return_type: TypeHint::parse(stream)?,
247            })
248        } else {
249            None
250        };
251
252        let semicolon = stream.expect_match(TokenKind::Symbol, &[";"])?;
253
254        Ok(Some(ExternalFunction {
255            extern_fn_token,
256            fn_token,
257            name,
258            opening_paren,
259            arguments,
260            closing_paren,
261            return_decl,
262            semicolon,
263        }))
264    }
265}
266
267impl Function {
268    fn try_parse<'s>(stream: &mut TokenStream<'s>) -> Result<Option<Self>, ParserError> {
269        let Some(fn_token) = stream.take_match(TokenKind::Identifier, &["fn"]) else {
270            return Ok(None);
271        };
272
273        let name = stream.expect_match(TokenKind::Identifier, &[])?;
274        let opening_paren = stream.expect_match(TokenKind::Symbol, &["("])?;
275
276        let mut arguments = Vec::new();
277        while let Some(arg_name) = stream.take_match(TokenKind::Identifier, &[]) {
278            let colon = stream.expect_match(TokenKind::Symbol, &[":"])?;
279            let reference_token = stream.take_match(TokenKind::Symbol, &["&"]);
280            let type_token = TypeHint::parse(stream)?;
281
282            arguments.push(FunctionArgument {
283                name: arg_name,
284                colon,
285                reference_token,
286                arg_type: type_token,
287            });
288
289            if stream.take_match(TokenKind::Symbol, &[","]).is_none() {
290                break;
291            }
292        }
293
294        let closing_paren = stream.expect_match(TokenKind::Symbol, &[")"])?;
295
296        let return_decl = if let Some(return_token) = stream.take_match(TokenKind::Symbol, &["->"])
297        {
298            Some(ReturnDecl {
299                return_token,
300                return_type: TypeHint::parse(stream)?,
301            })
302        } else {
303            None
304        };
305
306        let body = Body::parse(stream)?;
307
308        Ok(Some(Function {
309            fn_token,
310            name,
311            opening_paren,
312            arguments,
313            closing_paren,
314            return_decl,
315            body,
316        }))
317    }
318}
319
320impl Body {
321    fn parse<'s>(stream: &mut TokenStream<'s>) -> Result<Self, ParserError> {
322        let opening_brace = stream.expect_match(TokenKind::Symbol, &["{"])?;
323
324        let mut body = Vec::new();
325        while Statement::matches(stream) {
326            body.push(Statement::parse(stream)?);
327        }
328
329        let closing_brace = stream.expect_match(TokenKind::Symbol, &["}"])?;
330
331        Ok(Body {
332            opening_brace,
333            statements: body,
334            closing_brace,
335        })
336    }
337}
338
339impl TypeHint {
340    fn parse<'s>(stream: &mut TokenStream<'s>) -> Result<Self, ParserError> {
341        let type_name = stream.expect_match(TokenKind::Identifier, &[])?;
342
343        Ok(TypeHint { type_name })
344    }
345}
346
347impl Statement {
348    fn matches(stream: &mut TokenStream<'_>) -> bool {
349        !stream.end() && stream.peek_match(TokenKind::Symbol, &["}"]).is_none()
350    }
351
352    fn parse<'s>(stream: &mut TokenStream<'s>) -> Result<Self, ParserError> {
353        if let Some(return_token) = stream.take_match(TokenKind::Identifier, &["return"]) {
354            if let Some(semicolon) = stream.take_match(TokenKind::Symbol, &[";"]) {
355                return Ok(Statement::EmptyReturn(EmptyReturn {
356                    return_token,
357                    semicolon,
358                }));
359            }
360
361            let expr = Expression::parse(stream)?;
362            let semicolon = stream.expect_match(TokenKind::Symbol, &[";"])?;
363            return Ok(Statement::Return(ReturnWithValue {
364                return_token,
365                expression: expr,
366                semicolon,
367            }));
368        }
369
370        if let Some(decl_token) = stream.take_match(TokenKind::Identifier, &["var"]) {
371            let identifier = stream.expect_match(TokenKind::Identifier, &[])?;
372
373            let type_token = if stream.take_match(TokenKind::Symbol, &[":"]).is_some() {
374                Some(TypeHint::parse(stream)?)
375            } else {
376                None
377            };
378
379            let equals_token = stream.expect_match(TokenKind::Symbol, &["="])?;
380            let expression = Expression::parse(stream)?;
381            let semicolon = stream.expect_match(TokenKind::Symbol, &[";"])?;
382
383            return Ok(Statement::VariableDefinition(VariableDefinition {
384                decl_token,
385                identifier,
386                type_token,
387                equals_token,
388                initializer: expression,
389                semicolon,
390            }));
391        }
392
393        if let Some(if_token) = stream.take_match(TokenKind::Identifier, &["if"]) {
394            let condition = Expression::parse(stream)?;
395            let body = Body::parse(stream)?;
396
397            let else_branch =
398                if let Some(else_token) = stream.take_match(TokenKind::Identifier, &["else"]) {
399                    let else_body = Body::parse(stream)?;
400
401                    Some(Else {
402                        else_token,
403                        else_body,
404                    })
405                } else {
406                    None
407                };
408
409            return Ok(Statement::If(If {
410                if_token,
411                condition,
412                body,
413                else_branch,
414            }));
415        }
416
417        if let Some(loop_token) = stream.take_match(TokenKind::Identifier, &["loop"]) {
418            let body = Body::parse(stream)?;
419            return Ok(Statement::Loop(Loop { loop_token, body }));
420        }
421
422        if let Some(while_token) = stream.take_match(TokenKind::Identifier, &["while"]) {
423            // Desugar while into loop { if condition { loop_body; } else { break; } }
424            let condition = Expression::parse(stream)?;
425            let body = Body::parse(stream)?;
426            return Ok(Statement::Loop(Loop {
427                loop_token: while_token,
428                body: Body {
429                    opening_brace: body.opening_brace,
430                    closing_brace: body.closing_brace,
431                    statements: vec![Statement::If(If {
432                        if_token: while_token,
433                        condition: condition.clone(),
434                        body: body.clone(),
435                        else_branch: Some(Else {
436                            else_token: while_token,
437                            else_body: Body {
438                                opening_brace: body.opening_brace,
439                                closing_brace: body.closing_brace,
440                                statements: vec![Statement::Break(Break {
441                                    break_token: while_token,
442                                    semicolon: while_token,
443                                })],
444                            },
445                        }),
446                    })],
447                },
448            }));
449        }
450
451        if let Some(break_token) = stream.take_match(TokenKind::Identifier, &["break"]) {
452            let semicolon = stream.expect_match(TokenKind::Symbol, &[";"])?;
453            return Ok(Statement::Break(Break {
454                break_token,
455                semicolon,
456            }));
457        }
458        if let Some(continue_token) = stream.take_match(TokenKind::Identifier, &["continue"]) {
459            let semicolon = stream.expect_match(TokenKind::Symbol, &[";"])?;
460            return Ok(Statement::Continue(Continue {
461                continue_token,
462                semicolon,
463            }));
464        }
465
466        let expression = Expression::parse(stream)?;
467        let semicolon = stream.expect_match(TokenKind::Symbol, &[";"])?;
468        Ok(Statement::Expression {
469            expression,
470            semicolon,
471        })
472    }
473}
474
475impl<T> Literal<T>
476where
477    T: TypeSet,
478{
479    fn parse<'s>(stream: &mut TokenStream<'s>) -> Result<Self, ParserError> {
480        let token = stream.peek()?;
481
482        let token_source = stream.source(token.location);
483        let location = token.location;
484
485        let literal_value = match token.kind {
486            TokenKind::BinaryInteger => Self::parse_integer_literal(&token_source[2..], 2)
487                .map_err(|_| stream.error("Invalid binary integer literal"))?,
488            TokenKind::DecimalInteger => Self::parse_integer_literal(token_source, 10)
489                .map_err(|_| stream.error("Invalid integer literal"))?,
490            TokenKind::HexInteger => Self::parse_integer_literal(&token_source[2..], 16)
491                .map_err(|_| stream.error("Invalid hexadecimal integer literal"))?,
492            TokenKind::Float => <T::Float as FloatParser>::parse(token_source)
493                .map(LiteralValue::Float)
494                .map_err(|_| stream.error("Invalid float literal"))?,
495            TokenKind::String => match unescape(&token_source[1..token_source.len() - 1]) {
496                Ok(string) => LiteralValue::String(string),
497                Err(offset) => {
498                    return Err(ParserError {
499                        error: "Invalid escape sequence in string literal".to_string(),
500                        location: Location {
501                            start: token.location.start + offset,
502                            end: token.location.start + offset + 1,
503                        },
504                    });
505                }
506            },
507            TokenKind::Identifier if token_source == "true" => LiteralValue::Boolean(true),
508            TokenKind::Identifier if token_source == "false" => LiteralValue::Boolean(false),
509            _ => return Err(stream.error("Expected literal (number, string, or boolean)")),
510        };
511
512        stream.expect_match(token.kind, &[])?;
513        Ok(Self {
514            value: literal_value,
515            location,
516        })
517    }
518
519    fn parse_integer_literal(
520        token_source: &str,
521        radix: u32,
522    ) -> Result<LiteralValue<T>, ParseIntError> {
523        <T::Integer as IntParser>::parse(token_source, radix).map(LiteralValue::Integer)
524    }
525}
526
527fn unescape(s: &str) -> Result<String, usize> {
528    let mut result = String::new();
529    let mut escaped = false;
530    for (i, c) in s.char_indices().peekable() {
531        if escaped {
532            match c {
533                'n' => result.push('\n'),
534                't' => result.push('\t'),
535                '\\' => result.push('\\'),
536                '"' => result.push('"'),
537                '\'' => result.push('\''),
538                _ => return Err(i), // Invalid escape sequence
539            }
540            escaped = false;
541        } else if c == '\\' {
542            escaped = true;
543        } else {
544            result.push(c);
545        }
546    }
547
548    Ok(result)
549}
550
551enum Associativity {
552    Left,
553    None,
554}
555
556impl<T> Expression<T>
557where
558    T: TypeSet,
559{
560    fn parse<'s>(stream: &mut TokenStream<'s>) -> Result<Self, ParserError> {
561        // We define the binary operators from the lowest precedence to the highest.
562        // Each recursive call to `parse_binary` will handle one level of precedence, and pass
563        // the rest to the inner calls of `parse_binary`.
564        let operators: &[(Associativity, &[&str])] = &[
565            (Associativity::None, &["="]),
566            (Associativity::Left, &["||"]),
567            (Associativity::Left, &["&&"]),
568            (Associativity::Left, &["<", "<=", ">", ">=", "==", "!="]),
569            (Associativity::Left, &["|"]),
570            (Associativity::Left, &["^"]),
571            (Associativity::Left, &["&"]),
572            (Associativity::Left, &["<<", ">>"]),
573            (Associativity::Left, &["+", "-"]),
574            (Associativity::Left, &["*", "/"]),
575        ];
576
577        Self::parse_binary(stream, operators)
578    }
579
580    fn parse_binary<'s>(
581        stream: &mut TokenStream<'s>,
582        binary_operators: &[(Associativity, &[&str])],
583    ) -> Result<Self, ParserError> {
584        let Some(((associativity, current), higher)) = binary_operators.split_first() else {
585            unreachable!("At least one operator set is expected");
586        };
587
588        let mut expr = if higher.is_empty() {
589            Self::parse_unary(stream)?
590        } else {
591            Self::parse_binary(stream, higher)?
592        };
593
594        while let Some(operator) = stream.take_match(TokenKind::Symbol, current) {
595            let rhs = if higher.is_empty() {
596                Self::parse_unary(stream)?
597            } else {
598                Self::parse_binary(stream, higher)?
599            };
600
601            expr = Self::BinaryOperator {
602                name: operator,
603                operands: Box::new([expr, rhs]),
604            };
605
606            if matches!(associativity, Associativity::None) {
607                break;
608            }
609        }
610
611        Ok(expr)
612    }
613
614    fn parse_unary<'s>(stream: &mut TokenStream<'s>) -> Result<Self, ParserError> {
615        const UNARY_OPERATORS: &[&str] = &["!", "-", "&", "*"];
616        if let Some(operator) = stream.take_match(TokenKind::Symbol, UNARY_OPERATORS) {
617            let operand = Self::parse_unary(stream)?;
618            Ok(Self::UnaryOperator {
619                name: operator,
620                operand: Box::new(operand),
621            })
622        } else {
623            Self::parse_primary(stream)
624        }
625    }
626
627    fn parse_primary<'s>(stream: &mut TokenStream<'s>) -> Result<Self, ParserError> {
628        let token = stream.peek()?;
629
630        match token.kind {
631            TokenKind::Identifier => {
632                // true, false?
633                if let Ok(literal) = Literal::<T>::parse(stream) {
634                    return Ok(Self::Literal { value: literal });
635                }
636
637                Self::parse_call(stream)
638            }
639            TokenKind::Symbol if stream.source(token.location) == "(" => {
640                stream.take_match(token.kind, &[]).unwrap();
641                let expr = Self::parse(stream)?;
642                stream.expect_match(TokenKind::Symbol, &[")"])?;
643                Ok(expr)
644            }
645            TokenKind::HexInteger
646            | TokenKind::DecimalInteger
647            | TokenKind::BinaryInteger
648            | TokenKind::Float
649            | TokenKind::String => Literal::<T>::parse(stream).map(|value| Self::Literal { value }),
650            _ => Err(stream.error("Expected variable, literal, or '('")),
651        }
652    }
653
654    fn parse_call<'s>(stream: &mut TokenStream<'s>) -> Result<Self, ParserError> {
655        let token = stream.expect_match(TokenKind::Identifier, &[]).unwrap();
656
657        if stream.take_match(TokenKind::Symbol, &["("]).is_none() {
658            return Ok(Self::Variable { variable: token });
659        };
660
661        let mut arguments = Vec::new();
662        while stream.peek_match(TokenKind::Symbol, &[")"]).is_none() {
663            let arg = Self::parse(stream)?;
664            arguments.push(arg);
665
666            if stream.take_match(TokenKind::Symbol, &[","]).is_none() {
667                break;
668            }
669        }
670        stream.expect_match(TokenKind::Symbol, &[")"])?;
671
672        Ok(Self::FunctionCall {
673            name: token,
674            arguments: arguments.into_boxed_slice(),
675        })
676    }
677}
678
679struct TokenStream<'s> {
680    source: &'s str,
681    tokens: &'s [Token],
682    position: usize,
683}
684
685impl<'s> TokenStream<'s> {
686    fn new(source: &'s str, tokens: &'s [Token]) -> Self {
687        TokenStream {
688            source,
689            tokens,
690            position: 0,
691        }
692    }
693
694    fn end(&self) -> bool {
695        // Skip comments
696        let mut position = self.position;
697        while self.tokens.get(position).is_some()
698            && self.tokens[position].kind == TokenKind::Comment
699        {
700            position += 1;
701        }
702
703        position >= self.tokens.len()
704    }
705
706    fn peek(&mut self) -> Result<Token, ParserError> {
707        // Skip comments
708        let mut position = self.position;
709        while self.tokens.get(position).is_some()
710            && self.tokens[position].kind == TokenKind::Comment
711        {
712            position += 1;
713        }
714
715        if position < self.tokens.len() {
716            Ok(self.tokens[position])
717        } else {
718            Err(ParserError {
719                error: "Unexpected end of input".to_string(),
720                location: self.tokens.get(self.position).map_or(
721                    Location {
722                        start: self.source.len(),
723                        end: self.source.len(),
724                    },
725                    |t| t.location,
726                ),
727            })
728        }
729    }
730
731    fn peek_match(&mut self, token_kind: TokenKind, source: &[&str]) -> Option<Token> {
732        let token = self.peek().ok()?;
733
734        if token.kind == token_kind
735            && (source.is_empty() || source.contains(&self.source(token.location)))
736        {
737            Some(token)
738        } else {
739            None
740        }
741    }
742
743    fn take_match(&mut self, token_kind: TokenKind, source: &[&str]) -> Option<Token> {
744        // Skip comments
745        while self.tokens.get(self.position).is_some()
746            && self.tokens[self.position].kind == TokenKind::Comment
747        {
748            self.position += 1;
749        }
750
751        if let Some(token) = self.peek_match(token_kind, source) {
752            self.position += 1;
753            Some(token)
754        } else {
755            None
756        }
757    }
758
759    /// Takes the next token if it matches the expected kind and source.
760    /// Returns an error if the token does not match.
761    ///
762    /// If `source` is empty, it only checks the token kind.
763    /// If `source` is not empty, it checks if the token's source matches any of the provided strings.
764    fn expect_match(
765        &mut self,
766        token_kind: TokenKind,
767        source: &[&str],
768    ) -> Result<Token, ParserError> {
769        if let Some(token) = self.take_match(token_kind, source) {
770            Ok(token)
771        } else {
772            let token = self.peek().ok();
773            let found = if let Some(token) = token {
774                if token.kind == token_kind {
775                    format!("found '{}'", self.source(token.location))
776                } else {
777                    format!("found {:?}", token.kind)
778                }
779            } else {
780                "reached end of input".to_string()
781            };
782            match source {
783                [] => Err(self.error(format!("Expected {token_kind:?}, {found}"))),
784                [s] => Err(self.error(format!("Expected '{s}', {found}"))),
785                _ => Err(self.error(format!("Expected one of {source:?}, {found}"))),
786            }
787        }
788    }
789
790    fn error(&self, message: impl ToString) -> ParserError {
791        ParserError {
792            error: message.to_string(),
793            location: self
794                .tokens
795                .get(self.position)
796                .map(|t| t.location)
797                .unwrap_or(Location {
798                    start: self.source.len(),
799                    end: self.source.len(),
800                }),
801        }
802    }
803
804    fn source(&self, location: Location) -> &'s str {
805        location.extract(self.source)
806    }
807}
808
809pub fn parse<'s>(source: &'s str, tokens: &'s [Token]) -> Result<Program, ParserError> {
810    let mut stream = TokenStream::new(source, tokens);
811
812    Program::parse(&mut stream)
813}
814
815pub fn parse_expression<'s, T>(
816    source: &'s str,
817    tokens: &'s [Token],
818) -> Result<Expression<T>, ParserError>
819where
820    T: TypeSet,
821{
822    let mut stream = TokenStream::new(source, tokens);
823
824    Expression::<T>::parse(&mut stream)
825}
826
827#[cfg(test)]
828mod tests {
829    use super::*;
830
831    #[test]
832    fn test_unescape() {
833        assert_eq!(unescape(r#"Hello\nWorld\t!"#).unwrap(), "Hello\nWorld\t!");
834        assert_eq!(unescape(r#"Hello\\World"#).unwrap(), "Hello\\World");
835        assert_eq!(unescape(r#"Hello\zWorld"#), Err(6)); // Invalid escape sequence
836    }
837
838    #[test]
839    fn test_out_of_range_literal() {
840        let source = "0x100000000";
841        let tokens = vec![Token {
842            kind: TokenKind::HexInteger,
843            location: Location {
844                start: 0,
845                end: source.len(),
846            },
847        }];
848
849        let result =
850            parse_expression::<TypeSet32>(source, &tokens).expect_err("Parsing should fail");
851        assert_eq!("Invalid hexadecimal integer literal", result.error);
852    }
853}
somni_parser/parser.rs

somni_parser/
parser.rs
