1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
use crate::config::LexerConfig as Config;
use super::{
Kind, Lexer,
search::{SafeByteMatchTable, byte_search, safe_byte_match_table},
};
static NOT_REGULAR_WHITESPACE_TABLE: SafeByteMatchTable =
safe_byte_match_table!(|b| !matches!(b, b' ' | b'\t'));
static NOT_REGULAR_WHITESPACE_OR_LINE_BREAK_TABLE: SafeByteMatchTable =
safe_byte_match_table!(|b| !matches!(b, b' ' | b'\t' | b'\r' | b'\n'));
impl<C: Config> Lexer<'_, C> {
pub(super) fn whitespace_handler(&mut self) -> Kind {
// Single spaces between tokens are consumed directly by `read_next_token`.
// Scan additional spaces and tabs in one pass to avoid repeated token-loop
// checks, token-start updates, and indirect byte-handler calls.
// Leave line breaks and irregular whitespace to their handlers, which
// update newline state and record trivia.
byte_search! {
lexer: self,
table: NOT_REGULAR_WHITESPACE_TABLE,
handle_eof: 0,
};
Kind::Skip
}
/// Skip regular whitespace after a line break, returning the next byte or `None` at EOF.
pub(super) fn line_break_handler(&mut self) -> Option<u8> {
self.pending.set_is_on_new_line();
self.trivia_builder.handle_newline(&mut self.pending);
// Indentation is common after a line break.
// Consume it, along with any further line breaks.
// Irregular line breaks and whitespace are not consumed.
// They're uncommon, so leave them for the next call to `handle_byte` to take care of.
let byte = byte_search! {
lexer: self,
table: NOT_REGULAR_WHITESPACE_OR_LINE_BREAK_TABLE,
handle_eof: return None,
};
Some(byte)
}
}