koi_parser/
combinator.rs

1use std::{
2    io::{Read, Seek, Write},
3    marker::PhantomData,
4    ops::Range,
5};
6
7use iok::ast_node;
8
9use crate::{
10    lexer::Token,
11    token_tree::{Block, TokenTree, TokenTreeBlock, TokenTreeToken},
12};
13
14#[ast_node]
15#[derive(Debug, Clone, PartialEq)]
16pub enum Stmt {
17    Let(LetSmt),
18    Item(Item),
19    Expr(Expr),
20}
21
22#[ast_node]
23#[derive(Debug, Clone, PartialEq)]
24pub struct LetSmt {
25    ident: String,
26    ty: String,
27    value: Expr,
28}
29
30//************************************************************************//
31
32#[ast_node]
33#[derive(Debug, Clone, PartialEq)]
34pub enum Lit {
35    Str(Str),
36    Int(Int),
37    Float(Float),
38    Bool(Bool),
39}
40
41#[ast_node]
42#[derive(Debug, Clone, PartialEq)]
43pub struct Str {
44    str: String,
45}
46
47#[ast_node]
48#[derive(Debug, Clone, PartialEq)]
49pub struct Int {
50    int: i64,
51}
52
53#[ast_node]
54#[derive(Debug, Clone, PartialEq)]
55pub struct Float {
56    float: f64,
57}
58
59#[ast_node]
60#[derive(Debug, Clone, PartialEq)]
61pub struct Bool {
62    bool: bool,
63}
64
65//************************************************************************//
66
67#[ast_node]
68#[derive(Debug, Clone, PartialEq)]
69pub enum Expr {
70    Binary(BinaryExpr),
71    FnCall(FnCallExpr),
72    Lit(Lit),
73    Ident(Ident),
74    Return(Box<Expr>),
75    Block(Vec<Stmt>),
76}
77
78#[ast_node]
79#[derive(Debug, Clone, PartialEq)]
80pub struct Ident {
81    ident: String,
82}
83
84#[ast_node]
85#[derive(Debug, Clone, PartialEq)]
86pub struct BinaryExpr {
87    lhs: Box<Expr>,
88    op: Op,
89    rhs: Box<Expr>,
90}
91
92#[ast_node]
93#[derive(Debug, Clone, PartialEq)]
94pub struct FnCallExpr {
95    name: String,
96    args: Vec<Expr>,
97}
98
99//************************************************************************//
100
101#[ast_node]
102#[derive(Debug, Clone, PartialEq)]
103pub enum Item {
104    Enum(EnumItem),
105    Fn(FnItem),
106    Struct(StructItem),
107    Use(UsePath),
108}
109
110#[ast_node]
111#[derive(Debug, Clone, PartialEq)]
112pub struct EnumItem {
113    name: String,
114    variants: Vec<EnumVariant>,
115}
116
117#[ast_node]
118#[derive(Debug, Clone, PartialEq)]
119pub struct FnItem {
120    name: String,
121    params: Vec<String>,
122    body: Vec<Stmt>,
123}
124
125#[ast_node]
126#[derive(Debug, Clone, PartialEq)]
127pub struct StructItem {
128    ident: String,
129    fields: Vec<FieldItem>,
130}
131
132#[ast_node]
133#[derive(Debug, Clone, PartialEq)]
134pub struct FieldItem {
135    name: String,
136    ty: String,
137}
138
139#[ast_node]
140#[derive(Debug, Clone, PartialEq)]
141pub struct UsePath {
142    path: Vec<String>,
143}
144
145#[ast_node]
146#[derive(Debug, Clone, PartialEq)]
147pub struct EnumVariant {
148    name: String,
149    enum_or_struct_name: String,
150}
151
152#[ast_node]
153#[derive(Debug, Clone, PartialEq)]
154pub enum Op {
155    Plus,
156    Minus,
157    Star,
158    Slash,
159    Eq,
160    Neq,
161    And,
162    Or,
163    Lt,
164    Lte,
165    Gt,
166    Gte,
167}
168
169//************************************************************************//
170
171#[derive(Debug, bon::Builder)]
172pub struct ErrorMessage {
173    user_msg: Option<String>,
174    dev_msg: Option<String>,
175    user_location: Option<Vec<Range<usize>>>,
176    dev_location: Option<Vec<Range<usize>>>,
177}
178
179#[derive(Debug)]
180pub struct ParseError {
181    pub errors: Vec<ErrorMessage>,
182}
183
184impl ParseError {
185    pub fn new(error: ErrorMessage) -> Self {
186        Self {
187            errors: vec![error],
188        }
189    }
190
191    pub fn new_root_error(&mut self, error: ErrorMessage) {
192        self.errors.insert(0, error);
193    }
194    pub fn add_error(&mut self, error: ErrorMessage) {
195        self.errors.push(error);
196    }
197    pub fn add_if_error(&mut self, error: Option<ParseError>) {
198        if let Some(error) = error {
199            for error in error.errors {
200                self.errors.push(error);
201            }
202        }
203    }
204    pub fn new_root_if_error(&mut self, error: Option<ParseError>) {
205        if let Some(error) = error {
206            for error in error.errors {
207                self.errors.insert(0, error);
208            }
209        }
210    }
211}
212
213/// Cursor for recursive descent
214#[derive(Debug, Clone)]
215pub struct Cursor<'a> {
216    last: Option<&'a TokenTree>,
217    /// A stack of levels of the the tree. Each slice represents a level in the tree.
218    levels_stack: Vec<&'a [TokenTree]>,
219    /// position at each slice/level
220    levels_indices: Vec<usize>,
221}
222
223impl<'a> Cursor<'a> {
224    pub fn new(trees: &'a [TokenTree]) -> Self {
225        Cursor {
226            last: None,
227            levels_stack: vec![trees],
228            levels_indices: vec![0],
229        }
230    }
231
232    // todo add peekable that creates a fork
233    /// Peek the next token without moving the cursor forward
234    pub fn peek(&self) -> Option<&'a TokenTree> {
235        let mut peeking_level = 1;
236        let levels_len = self.levels_stack.len();
237        debug_assert_eq!(levels_len, self.levels_indices.len());
238        if levels_len == 0 {
239            return None;
240        }
241        loop {
242            let level_index = levels_len - peeking_level;
243            let level = self.levels_stack[level_index];
244            let index_in_level = self.levels_indices[level_index];
245            if index_in_level >= level.len() {
246                // Go up
247                if level_index == 0 {
248                    // at the end
249                    return None;
250                }
251                peeking_level += 1;
252                continue;
253            }
254            return Some(&level[index_in_level]);
255        }
256    }
257
258    pub fn peek_token(&self) -> Option<&'a TokenTreeToken> {
259        self.peek().and_then(|e| match e {
260            TokenTree::Token(token) => Some(token),
261            TokenTree::Block { .. } => None,
262        })
263    }
264
265    pub fn peek_block(&self) -> Option<&'a TokenTreeBlock> {
266        self.peek().and_then(|e| match e {
267            TokenTree::Token { .. } => None,
268            TokenTree::Block(block) => Some(block),
269        })
270    }
271
272    /// Advances the cursor and returns the next value.
273    pub fn next(&mut self) -> Option<&'a TokenTree> {
274        loop {
275            let level = match self.levels_stack.last() {
276                Some(s) => s,
277                None => return None,
278            };
279            let idx = self.levels_indices.last_mut().unwrap();
280            if *idx >= level.len() {
281                // Finished this level, pop
282                self.levels_stack.pop();
283                self.levels_indices.pop();
284                continue;
285            }
286
287            let item = &level[*idx];
288            self.last = Some(item);
289            *idx += 1;
290
291            // If it's a delimited group, descend
292            if let TokenTree::Block(block) = item {
293                if !block.trees.is_empty() {
294                    self.levels_stack.push(&*block.trees);
295                    self.levels_indices.push(0);
296                }
297            }
298
299            return Some(item);
300        }
301    }
302
303    pub fn next_token(&mut self) -> Option<&'a TokenTreeToken> {
304        self.next().and_then(|e| match e {
305            TokenTree::Token(token) => Some(token),
306            TokenTree::Block { .. } => None,
307        })
308    }
309
310    pub fn next_block(&mut self) -> Option<&'a TokenTreeBlock> {
311        self.next().and_then(|e| match e {
312            TokenTree::Token { .. } => None,
313            TokenTree::Block(block) => Some(block),
314        })
315    }
316
317    /// Skip the next block to not descent down into
318    pub fn skip_next_block_of(&mut self, delimiter: Block) -> Option<()> {
319        if let Some(tt) = self.peek()
320            && let TokenTree::Block(block) = tt
321            && block.delimiter == delimiter
322        {
323            loop {
324                let level = match self.levels_stack.last() {
325                    Some(s) => s,
326                    None => return None,
327                };
328                let idx = self.levels_indices.last_mut().unwrap();
329                let item = &level[*idx];
330                self.last = Some(item);
331                if *idx >= level.len() {
332                    // Finished this level, pop
333                    self.levels_stack.pop();
334                    self.levels_indices.pop();
335                    continue;
336                }
337                let item = &level[*idx];
338                self.last = Some(item);
339                *idx += 1;
340
341                debug_assert!(item.block_of(delimiter).is_some());
342                return Some(());
343            }
344        }
345        None
346    }
347
348    pub fn enter_block_of(&mut self, delimiter: Block) -> Option<&'a TokenTreeBlock> {
349        if let Some(tt) = self.peek()
350            && let TokenTree::Block(block) = tt
351            && block.delimiter == delimiter
352        {
353            let _ = self.next().unwrap(); // consume block
354            return Some(block);
355        }
356        None
357    }
358
359    /// Yields the last value emitted by [`next`]
360    pub fn last(&self) -> Option<&'a TokenTree> {
361        self.last
362    }
363
364    /// True if end of data
365    pub fn eod(&self) -> bool {
366        self.peek().is_none()
367    }
368
369    pub fn fork(&self) -> Self {
370        self.clone()
371    }
372
373    /// Get the location of the cursor
374    pub fn last_location(&self) -> Range<usize> {
375        if let Some(token) = self.last() {
376            token.span()
377        } else if self.eod() {
378            usize::MAX..usize::MAX // todo something better
379        } else {
380            0..0 // is empty
381        }
382    }
383
384    pub fn next_location(&self) -> Range<usize> {
385        if let Some(token) = self.peek() {
386            token.span()
387        } else if self.eod() {
388            usize::MAX..usize::MAX // todo something better
389        } else {
390            0..0 // is empty
391        }
392    }
393}
394
395#[derive(Debug)]
396pub struct ParseStream<'a, State = ()> {
397    pub cursor: Cursor<'a>,
398    pub state: &'a mut State,
399}
400
401impl<'a, State> ParseStream<'a, State> {
402    pub fn new(trees: &'a [TokenTree], state: &'a mut State) -> Self {
403        ParseStream {
404            cursor: Cursor::new(trees),
405            state,
406        }
407    }
408
409    //************************************************************************//
410
411    pub fn parse<T: Parser<State>>(&mut self) -> Result<T, ParseError> {
412        let checkpoint = self.cursor.fork();
413        match T::parse(self) {
414            Ok(value) => return Ok(value),
415            Err(error) => {
416                self.cursor = checkpoint;
417                return Err(error);
418            }
419        }
420    }
421
422    pub fn parse_punctuated<T: Parser<State>>(
423        &mut self,
424        punctuation: &Token,
425        // Require at least one value in vec or error
426        require_one: bool,
427        // Will parse to the end of the stream if true and error if cannot.
428        // If false, it does not have to parse to the end before returning the already
429        // parsed values.
430        take_all: bool,
431        // How to handle trailing punctuation
432        trailing: Trailing,
433    ) -> Result<Vec<T>, ParseError> {
434        let start = self.cursor.fork();
435        let mut checkpoint;
436        let mut out = Vec::new();
437        let mut parsing_error = None;
438        loop {
439            checkpoint = self.cursor.fork();
440            match T::parse(self) {
441                Ok(value) => out.push(value),
442                Err(err) => {
443                    parsing_error = Some(err);
444                    break;
445                }
446            }
447            if let Some(tt) = self.cursor.peek()
448                && let Some(token) = tt.token()
449                && &token.token == punctuation
450            {
451                self.cursor.next().unwrap();
452                continue;
453            } else {
454                if matches!(trailing, Trailing::Required) {
455                    let location = vec![self.cursor.last_location()];
456                    self.cursor = start;
457                    return Err(ParseError::new(
458                        ErrorMessage::builder()
459                            .user_msg(format!("Expected trailing punctuation `{:?}`", punctuation))
460                            .user_location(location)
461                            .build(),
462                    ));
463                }
464                break;
465            }
466        }
467        if take_all && !self.cursor.eod() {
468            let location = vec![self.cursor.last_location()];
469            let mut error = ParseError::new(
470                ErrorMessage::builder()
471                    .user_msg(format!(
472                        "All punctuated types of `{punctuation:?}` are not the same"
473                    ))
474                    .user_location(location)
475                    .build(),
476            ); // todo impl display for punct
477            error.new_root_if_error(parsing_error);
478            self.cursor = start;
479            return Err(error);
480        }
481        let has_trailing = parsing_error.is_some();
482        if has_trailing && matches!(trailing, Trailing::Disallow) {
483            let location = vec![self.cursor.last_location()];
484            self.cursor = start;
485            return Err(ParseError::new(
486                ErrorMessage::builder()
487                    .user_msg(format!("Trailing punctuation `{:?}`", punctuation))
488                    .user_location(location)
489                    .build(),
490            ));
491        }
492        if require_one && out.is_empty() {
493            let location = vec![start.last_location(), self.cursor.last_location()];
494            self.cursor = start;
495            return Err(ParseError::new(
496                ErrorMessage::builder()
497                    .user_msg("Expected at least one value".to_string())
498                    .user_location(location)
499                    .build(),
500            ));
501        }
502
503        Ok(out)
504    }
505
506    /// If the next item in the cursor is the type of block, turn the block into a parse stream
507    pub fn parse_block<T: Parser<State>>(&mut self, delimiter: Block) -> Result<T, ParseError> {
508        if let Some(tt) = self.cursor.peek()
509            && let TokenTree::Block(block) = tt
510        {
511            if delimiter == block.delimiter {
512                let checkpoint = self.cursor.fork();
513                self.cursor.next().unwrap(); // consume block
514                match T::parse(self) {
515                    Ok(value) => return Ok(value),
516                    Err(error) => {
517                        self.cursor = checkpoint;
518                        return Err(error);
519                    }
520                }
521            } else {
522                Err(ParseError::new(
523                    ErrorMessage::builder()
524                        .user_msg(format!(
525                            "Expected block of `{}`, found `{:?}`",
526                            delimiter, block.delimiter
527                        ))
528                        .user_location(vec![self.cursor.next_location()])
529                        .build(),
530                ))
531            }
532        } else {
533            Err(ParseError::new(
534                ErrorMessage::builder()
535                    .user_msg(format!("Expected block of `{}`", delimiter))
536                    .user_location(vec![self.cursor.next_location()])
537                    .build(),
538            ))
539        }
540    }
541
542    pub fn parse_block_punctuated<T: Parser<State>>(
543        &mut self,
544        delimiter: Block,
545        punctuation: &Token,
546        require_one: bool,
547        take_all: bool,
548        trailing: Trailing,
549    ) -> Result<Vec<T>, ParseError> {
550        if let Some(tt) = self.cursor.peek()
551            && let TokenTree::Block(block) = tt
552        {
553            if delimiter == block.delimiter {
554                let checkpoint = self.cursor.fork();
555                self.cursor.next().unwrap(); // consume block
556                let result = self.parse_punctuated(punctuation, require_one, take_all, trailing);
557                match result {
558                    Ok(value) => return Ok(value),
559                    Err(error) => {
560                        self.cursor = checkpoint;
561                        return Err(error);
562                    }
563                }
564            } else {
565                Err(ParseError::new(
566                    ErrorMessage::builder()
567                        .user_msg(format!(
568                            "Expected block of `{}`, found `{:?}`",
569                            delimiter, block.delimiter
570                        ))
571                        .user_location(vec![self.cursor.next_location()])
572                        .build(),
573                ))
574            }
575        } else {
576            Err(ParseError::new(
577                ErrorMessage::builder()
578                    .user_msg(format!("Expected block of `{}`", delimiter))
579                    .user_location(vec![self.cursor.next_location()])
580                    .build(),
581            ))
582        }
583    }
584
585    /// Turns the next block into a new parse stream. Useful since will only parse inside the block and below.
586    /// Unlike using [`Cursor`] which moves in and out of blocks.
587    /// After parsing a successful block. You may need to move your cursor end
588    pub fn new_block_parse_stream<'b>(
589        &'b mut self,
590        delimiter: Block,
591    ) -> Result<ParseStream<'b, State>, ParseError> {
592        if let Some(tt) = self.cursor.peek()
593            && let TokenTree::Block(block) = tt
594        {
595            if delimiter == block.delimiter {
596                Ok(ParseStream {
597                    cursor: Cursor::new(&block.trees),
598                    state: self.state,
599                })
600            } else {
601                Err(ParseError::new(
602                    ErrorMessage::builder()
603                        .user_msg(format!(
604                            "Expected block of `{}`, found `{:?}`",
605                            delimiter, block.delimiter
606                        ))
607                        .user_location(vec![self.cursor.next_location()])
608                        .build(),
609                ))
610            }
611        } else {
612            Err(ParseError::new(
613                ErrorMessage::builder()
614                    .user_msg(format!("Expected block of `{}`", delimiter))
615                    .user_location(vec![self.cursor.next_location()])
616                    .build(),
617            ))
618        }
619    }
620}
621
622pub trait Parser<State = ()>: Sized {
623    fn parse(input: &mut ParseStream<State>) -> Result<Self, ParseError>;
624}
625
626impl Parser for Op {
627    fn parse(input: &mut ParseStream) -> Result<Self, ParseError> {
628        if let Some(tt) = input.cursor.peek()
629            && let Some(token) = tt.token()
630        {
631            match &token.token {
632                Token::Plus => {
633                    input.cursor.next();
634                    return Ok(Op::Plus(token.span.clone().into()));
635                }
636                Token::Minus => {
637                    input.cursor.next();
638                    return Ok(Op::Minus(token.span.clone().into()));
639                }
640                Token::Star => {
641                    input.cursor.next();
642                    return Ok(Op::Star(token.span.clone().into()));
643                }
644                Token::Slash => {
645                    input.cursor.next();
646                    return Ok(Op::Slash(token.span.clone().into()));
647                }
648                Token::Eq => {
649                    input.cursor.next();
650                    return Ok(Op::Eq(token.span.clone().into()));
651                }
652                Token::Neq => {
653                    input.cursor.next();
654                    return Ok(Op::Neq(token.span.clone().into()));
655                }
656                Token::And => {
657                    input.cursor.next();
658                    return Ok(Op::And(token.span.clone().into()));
659                }
660                Token::Or => {
661                    input.cursor.next();
662                    return Ok(Op::Or(token.span.clone().into()));
663                }
664                Token::Lt => {
665                    input.cursor.next();
666                    return Ok(Op::Lt(token.span.clone().into()));
667                }
668                Token::Lte => {
669                    input.cursor.next();
670                    return Ok(Op::Lte(token.span.clone().into()));
671                }
672                Token::Gt => {
673                    input.cursor.next();
674                    return Ok(Op::Gt(token.span.clone().into()));
675                }
676                Token::Gte => {
677                    input.cursor.next();
678                    return Ok(Op::Gte(token.span.clone().into()));
679                }
680                _ => {}
681            }
682        }
683        Err(ParseError::new(
684            ErrorMessage::builder()
685                .user_msg("Expected operator".to_string())
686                .user_location(vec![input.cursor.next_location()])
687                .build(),
688        ))
689    }
690}
691
692pub enum Trailing {
693    Allowed,
694    Required,
695    Disallow,
696}
697
698#[cfg(test)]
699mod tests {
700    use super::*;
701    use crate::{
702        lexer::{self, Token},
703        token_tree::TokenTreeParser,
704    };
705
706    macro_rules! parse_stream_setup {
707        ($source:ident, $parse_stream:ident) => {
708            let lex = lexer::lexer($source);
709            let token_tree = TokenTreeParser::new(lex.collect()).parse().unwrap();
710            let mut temp = ();
711            let mut $parse_stream = ParseStream::new(&token_tree, &mut temp);
712        };
713    }
714
715    #[test]
716    fn parse_punctuated_basic() {
717        let source = "+,-, *,/,";
718        parse_stream_setup!(source, parse_stream);
719        let ast = parse_stream
720            .parse_punctuated::<Op>(&Token::Comma, false, true, Trailing::Allowed)
721            .unwrap();
722
723        let expected_ast = vec![
724            Op::Plus((0..1).into()),
725            Op::Minus((2..3).into()),
726            Op::Star((5..6).into()),
727            Op::Slash((7..8).into()),
728        ];
729
730        assert_eq!(ast, expected_ast);
731    }
732
733    #[test]
734    fn parse_punctuated_empty_ok() {
735        let source = "";
736        parse_stream_setup!(source, parse_stream);
737        let ast = parse_stream
738            .parse_punctuated::<Op>(&Token::Comma, false, true, Trailing::Allowed)
739            .unwrap();
740        assert!(ast.is_empty());
741    }
742
743    #[test]
744    fn parse_punctuated_empty_require_one_error() {
745        let source = "";
746        parse_stream_setup!(source, parse_stream);
747        let err = parse_stream
748            .parse_punctuated::<Op>(&Token::Comma, true, true, Trailing::Allowed)
749            .unwrap_err();
750        assert!(
751            err.errors[0]
752                .user_msg
753                .as_ref()
754                .unwrap()
755                .contains("Expected at least one value")
756        );
757    }
758
759    #[test]
760    fn parse_punctuated_trailing_disallowed_error() {
761        let source = "+, -,";
762        parse_stream_setup!(source, parse_stream);
763        let err = parse_stream
764            .parse_punctuated::<Op>(&Token::Comma, false, true, Trailing::Disallow)
765            .unwrap_err();
766
767        assert!(
768            err.errors[0]
769                .user_msg
770                .as_ref()
771                .unwrap()
772                .contains("Trailing punctuation")
773        );
774    }
775
776    #[test]
777    fn parse_punctuated_missing_trailing_required_error() {
778        let source = "+, -";
779        parse_stream_setup!(source, parse_stream);
780        let err = parse_stream
781            .parse_punctuated::<Op>(&Token::Comma, false, true, Trailing::Required)
782            .unwrap_err();
783
784        assert!(
785            err.errors[0]
786                .user_msg
787                .as_ref()
788                .unwrap()
789                .contains("Expected trailing punctuation")
790        );
791    }
792
793    #[test]
794    fn parse_punctuated_take_all_error() {
795        let source = "+, -, *, / extra";
796        parse_stream_setup!(source, parse_stream);
797        let err = parse_stream
798            .parse_punctuated::<Op>(&Token::Comma, false, true, Trailing::Allowed)
799            .unwrap_err();
800        assert!(
801            err.errors[0]
802                .user_msg
803                .as_ref()
804                .unwrap()
805                .contains("All punctuated types")
806        );
807    }
808
809    #[test]
810    fn parse_punctuated_take_all_false_ok() {
811        let source = "+, -, *, / extra";
812        parse_stream_setup!(source, parse_stream);
813        let ast = parse_stream
814            .parse_punctuated::<Op>(&Token::Comma, false, false, Trailing::Allowed)
815            .unwrap();
816
817        let expected_ast = vec![
818            Op::Plus((0..1).into()),
819            Op::Minus((3..4).into()),
820            Op::Star((6..7).into()),
821            Op::Slash((9..10).into()),
822        ];
823
824        assert_eq!(ast, expected_ast);
825    }
826
827    #[test]
828    fn parse_punctuated_invalid_middle_token() {
829        let source = "+, foo, -";
830        parse_stream_setup!(source, parse_stream);
831        let err = parse_stream
832            .parse_punctuated::<Op>(&Token::Comma, false, true, Trailing::Allowed)
833            .unwrap_err();
834
835        assert!(
836            err.errors[0]
837                .user_msg
838                .as_ref()
839                .unwrap()
840                .contains("Expected operator")
841        );
842    }
843
844    #[test]
845    fn parse_punctuated_single_item_ok() {
846        let source = "+";
847        parse_stream_setup!(source, parse_stream);
848        let ast = parse_stream
849            .parse_punctuated::<Op>(&Token::Comma, true, true, Trailing::Allowed)
850            .unwrap();
851
852        assert_eq!(ast, vec![Op::Plus((0..1).into())]);
853    }
854
855    #[test]
856    fn parse_punctuated_single_item_trailing_allowed_ok() {
857        let source = "+,";
858        parse_stream_setup!(source, parse_stream);
859        let ast = parse_stream
860            .parse_punctuated::<Op>(&Token::Comma, true, true, Trailing::Allowed)
861            .unwrap();
862
863        assert_eq!(ast, vec![Op::Plus((0..1).into())]);
864    }
865
866    #[test]
867    fn parse_punctuated_whitespace_ok() {
868        let source = " + ,   - ,   * , / ";
869        parse_stream_setup!(source, parse_stream);
870        let ast = parse_stream
871            .parse_punctuated::<Op>(&Token::Comma, false, true, Trailing::Allowed)
872            .unwrap();
873
874        let expected_ast = vec![
875            Op::Plus((1..2).into()),
876            Op::Minus((7..8).into()),
877            Op::Star((13..14).into()),
878            Op::Slash((17..18).into()),
879        ];
880
881        assert_eq!(ast, expected_ast);
882    }
883
884    #[test]
885    fn block_nested() {
886        let source = "{ { +, - } }";
887        parse_stream_setup!(source, parse_stream);
888        parse_stream
889            .cursor
890            .enter_block_of(Block::Brace)
891            .expect("outer block");
892
893        parse_stream
894            .cursor
895            .enter_block_of(Block::Brace)
896            .expect("inner block");
897
898        let ast = parse_stream
899            .parse_punctuated::<Op>(&Token::Comma, false, true, Trailing::Allowed)
900            .unwrap();
901
902        let expected_ast = vec![Op::Plus((4..5).into()), Op::Minus((7..8).into())];
903
904        assert_eq!(ast, expected_ast);
905    }
906
907    #[test]
908    fn block_into_parse_stream_with_outer_tokens() {
909        let source = "+ { -, *, / } -";
910        parse_stream_setup!(source, parse_stream);
911
912        let first_op = parse_stream.parse::<Op>().unwrap();
913        assert_eq!(first_op, Op::Plus((0..1).into()));
914
915        let outside_block_fork = parse_stream.cursor.fork();
916        parse_stream
917            .cursor
918            .enter_block_of(Block::Brace)
919            .expect("Expected inner block");
920
921        let inside_block_fork = parse_stream.cursor.fork();
922        let _ = parse_stream
923            .parse_punctuated::<Op>(&Token::Comma, false, true, Trailing::Allowed)
924            .unwrap_err();
925        parse_stream.cursor = inside_block_fork;
926        let inner_ops = parse_stream
927            .parse_punctuated::<Op>(&Token::Comma, false, false, Trailing::Allowed)
928            .unwrap();
929
930        let expected_inner = vec![
931            Op::Minus((4..5).into()),
932            Op::Star((7..8).into()),
933            Op::Slash((10..11).into()),
934        ];
935        assert_eq!(inner_ops, expected_inner);
936
937        let trailing_op = parse_stream.parse::<Op>().unwrap();
938        assert_eq!(trailing_op, Op::Minus((14..15).into()));
939
940        assert!(parse_stream.cursor.eod());
941
942        parse_stream.cursor = outside_block_fork;
943
944        let inner_ops = parse_stream
945            .new_block_parse_stream(Block::Brace)
946            .unwrap()
947            .parse_punctuated::<Op>(&Token::Comma, false, true, Trailing::Allowed)
948            .unwrap();
949        assert_eq!(inner_ops, expected_inner);
950        parse_stream.cursor.skip_next_block_of(Block::Brace);
951
952        let trailing_op = parse_stream.parse::<Op>().unwrap();
953        assert_eq!(trailing_op, Op::Minus((14..15).into()));
954
955        assert!(parse_stream.cursor.eod());
956    }
957}
koi_parser/combinator.rs

koi_parser/
combinator.rs
