builtin_tokens/
builtin_tokens.rs

1use elyze::bytes::token::Token;
2use elyze::errors::{ParseError, ParseResult};
3use elyze::peek::{peek, DefaultPeekableImplementation, Last, PeekSize, PeekableImplementation};
4use elyze::recognizer::{recognize, Recognizer};
5use elyze::scanner::Scanner;
6use elyze::separated_list::{get_scanner_without_trailing_separator, SeparatedList};
7use elyze::visitor::Visitor;
8
9fn main() -> ParseResult<()> {
10    let data = b"+-*";
11
12    // use recognize
13    let mut scanner = Scanner::new(data);
14    let recognized = recognize(Token::Plus, &mut scanner)?;
15    assert_eq!(recognized, Token::Plus);
16
17    // use the recognizer
18    let mut scanner = Scanner::new(data);
19    let recognized = Recognizer::new(&mut scanner)
20        .try_or(Token::Dash)?
21        .try_or(Token::Plus)?
22        .try_or(Token::Star)?
23        .finish()
24        .ok_or(ParseError::UnexpectedToken)?;
25    assert_eq!(recognized, Token::Plus);
26
27    // use the visitor
28    let mut scanner = Scanner::new(data);
29    let accepted = Token::accept(&mut scanner)?;
30    assert_eq!(accepted, Token::Plus);
31
32    // use peek
33    let mut scanner = Scanner::new(data);
34    let peeked = peek(Token::Dash, &mut scanner)?;
35    if let Some(peeked) = peeked {
36        assert_eq!(peeked.peeked_slice(), b"+");
37    }
38
39    // last token
40    let data = b" 8 + ( 7 * ( 1 + 2 ) )";
41    let scanner = Scanner::new(data);
42    let peeked = peek(Last::new(Token::CloseParen), &scanner)?;
43    if let Some(peeked) = peeked {
44        assert_eq!(peeked.peeked_slice(), b" 8 + ( 7 * ( 1 + 2 ) ");
45    }
46
47    // separated list
48    // defines a structure to implement Peekable
49    // using the Visitor pattern excluding the comma token
50    struct AnyTokenExceptComma;
51
52    // Enable the Peekable trait using the Visitor pattern
53    impl PeekableImplementation for AnyTokenExceptComma {
54        type Type = DefaultPeekableImplementation;
55    }
56
57    // Define the PeekSize trait
58    impl PeekSize<u8> for AnyTokenExceptComma {}
59
60    // Define the Visitor trait for the AnyTokenExceptComma structure
61    // excluding the comma token
62    impl<'a> Visitor<'a, u8> for AnyTokenExceptComma {
63        fn accept(scanner: &mut Scanner<'a, u8>) -> ParseResult<Self> {
64            let token = Token::accept(scanner)?;
65            match token {
66                Token::Comma => Err(ParseError::UnexpectedToken),
67                _ => Ok(AnyTokenExceptComma),
68            }
69        }
70    }
71
72    // Define a structure to implement Visitor
73    #[derive(Debug, PartialEq)]
74    struct TokenData(Token);
75
76    impl<'a> Visitor<'a, u8> for TokenData {
77        fn accept(scanner: &mut Scanner<'a, u8>) -> ParseResult<Self> {
78            let token = Token::accept(scanner)?;
79            match token {
80                Token::Comma => Err(ParseError::UnexpectedToken),
81                _ => Ok(TokenData(token)),
82            }
83        }
84    }
85
86    // Define a structure to implement Visitor for the separator
87    struct SeparatorComma;
88
89    impl<'a> Visitor<'a, u8> for SeparatorComma {
90        fn accept(scanner: &mut Scanner<'a, u8>) -> ParseResult<Self> {
91            recognize(Token::Comma, scanner)?;
92            Ok(SeparatorComma)
93        }
94    }
95
96    let data = b"*,-,+,/,";
97    let scanner = Scanner::new(data);
98    // clean up the data of its trailing comma
99    let mut data_scanner =
100        get_scanner_without_trailing_separator(AnyTokenExceptComma, Token::Comma, &scanner)?;
101    assert_eq!(data_scanner.data(), b"*,-,+,/"); // data without a trailing comma
102    // accept the separated list
103    let list = SeparatedList::<u8, TokenData, SeparatorComma>::accept(&mut data_scanner)?;
104    assert_eq!(
105        list.data,
106        vec![
107            TokenData(Token::Star),
108            TokenData(Token::Dash),
109            TokenData(Token::Plus),
110            TokenData(Token::Slash),
111        ]
112    );
113
114    Ok(())
115}
builtin_tokens/builtin_tokens.rs

builtin_tokens/
builtin_tokens.rs
