1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
use crate::config::LexerConfig as Config;
use super::{
Kind, Lexer,
search::{SafeByteMatchTable, byte_search, safe_byte_match_table},
};
static NOT_REGULAR_WHITESPACE_TABLE: SafeByteMatchTable =
safe_byte_match_table!(|b| !matches!(b, b' ' | b'\t'));
static NOT_REGULAR_WHITESPACE_OR_LINE_BREAK_TABLE: SafeByteMatchTable =
safe_byte_match_table!(|b| !matches!(b, b' ' | b'\t' | b'\r' | b'\n'));
impl<C: Config> Lexer<'_, C> {
pub(super) fn whitespace_handler(&mut self) -> Kind {
// Single spaces between tokens are consumed directly by `read_next_token`.
// Scan additional spaces and tabs in one pass to avoid repeated token-loop
// checks, token-start updates, and indirect byte-handler calls.
// Leave line breaks and irregular whitespace to their handlers, which
// update newline state and record trivia.
byte_search! {
lexer: self,
table: NOT_REGULAR_WHITESPACE_TABLE,
handle_eof: 0,
};
Kind::Skip
}
pub(super) fn line_break_handler(&mut self) -> Kind {
self.token.set_is_on_new_line(true);
self.trivia_builder.handle_newline();
// Indentation is common after a line break.
// Consume it, along with any further line breaks.
// Irregular line breaks and whitespace are not consumed.
// They're uncommon, so leave them for the next call to `handle_byte` to take care of.
byte_search! {
lexer: self,
table: NOT_REGULAR_WHITESPACE_OR_LINE_BREAK_TABLE,
handle_eof: 0, // Fall through to below
};
Kind::Skip
}
}