impral/lexer/
groupenizer.rs

1//! The groupenizer takes a stream of tokens and converts it into a *tree* of token streams.
2use super::*;
3
4/// Find and stack groups from the given stream of tokens.
5pub fn groupenize(tokens: &mut PeekableTokenStream<impl TokenStream>, delimiter: Option<Symbol>) -> PeekableTokenStream<impl TokenStream + '_> {
6    std::iter::from_fn(move || {
7        match tokens.next() {
8            Some(Token {
9                content: TokenContent::Symbol(
10                    symbol @ (
11                        Symbol::ParenLeft |
12                        Symbol::CurlyLeft |
13                        Symbol::BraketLeft
14                    )
15                ),
16                start,
17                end
18            }) => {
19                let group: Vec<Token> = groupenize(tokens, symbol.get_delimiter()).collect();
20                let end = group.last().map(|t| t.end).unwrap_or_else(|| end+1);
21                let group = TokenContent::Group(symbol, group);
22                //tokens.next();
23                Some(Token {
24                    content: group,
25                    start,
26                    end
27                })
28            },
29            Some(Token {
30                content: TokenContent::Symbol(
31                    symbol
32                ),
33                ..
34            }) if delimiter.map(|d| d == symbol).unwrap_or(false) => {
35                None // end of current group
36            },
37            Some(token) => Some(token),
38            
39            // TODO: Check for unmatched delimiters by `if let None = delimiter`
40            None => None, // natural end
41        }
42    }).peekmore()
43}
impral/lexer/groupenizer.rs

impral/lexer/
groupenizer.rs
