parol_runtime/lexer/
mod.rs

1#![forbid(missing_docs)]
2
3///
4/// Type used for token types the user provides.
5///
6pub type TerminalIndex = u16;
7
8///
9/// Type used for token numbers in tokens.
10///
11pub type TokenNumber = u32;
12
13///
14/// Invalid token number
15/// These tokens have been inserted by the parser during error recovery.
16/// They have no valid index within the original token stream.
17/// ATTENTION: This could lead to invalid array index access.
18/// TODO:
19/// Maybe create a special token type that is used in error recovery scenarios and that can't
20/// be confused with normal tokens.
21/// Or ensure that such tokens never leave the parser.
22///
23pub const INVALID_TOKEN_NUMBER: TokenNumber = TokenNumber::MAX;
24
25///
26/// Module with common formatting trait
27///
28pub mod format_token;
29pub use format_token::FormatToken;
30
31///
32/// Module with a location type
33///
34pub mod location;
35pub use location::{Location, LocationBuilder};
36
37///
38/// Module to support handling of std::ops::Range
39///
40pub mod rng;
41pub use rng::{Span, ToSpan};
42
43///
44/// Module that provides basic token implementation.
45///
46pub mod token;
47pub use token::{BLOCK_COMMENT, EOI, FIRST_USER_TOKEN, LINE_COMMENT, NEW_LINE, Token, WHITESPACE};
48
49mod token_buffer;
50pub(crate) use token_buffer::TokenBuffer;
51
52///
53/// Module that provides the TokenIter type.
54///
55pub mod token_iter;
56pub use token_iter::TokenIter;
57
58///
59/// Module that provides the TokenStream type.
60///
61pub mod token_stream;
62pub use token_stream::TokenStream;
63
64///
65/// This is an  unmatchable regular expression.
66/// It is normally not included in the generated Regex's source but stands for
67/// tokens that should be skipped, i.e. if a language doesn't support block
68/// comments you could mark the regex on index token::BLOCK_COMMENT as
69/// unmatchable.
70///
71pub const UNMATCHABLE_TOKEN: &str = r"\w\b\w";
72
73///
74/// Regular expression for new lines
75///
76pub const NEW_LINE_TOKEN: &str = r"\r\n|\r|\n";
77
78///
79/// Regular expression for any whitespace except newline characters
80///
81pub const WHITESPACE_TOKEN: &str = r"[\s--\r\n]+";
82
83///
84/// Regular expression that matches any other token. With this you can detect
85/// so far unmatched tokens. It is only used for error detection during lexical
86/// analysis.
87///
88pub const ERROR_TOKEN: &str = r###"."###;
parol_runtime/lexer/mod.rs

parol_runtime/lexer/
mod.rs
