use super::{LexError, LexErrorKind, Lexer};
use crate::span::Span;
pub(crate) fn is_whitespace(ch: char) -> bool {
matches!(
ch,
'\u{0009}' | '\u{000b}' | '\u{000c}' | '\u{feff}' ) || is_space_separator(ch)
}
fn is_space_separator(ch: char) -> bool {
matches!(
ch,
'\u{0020}' | '\u{00a0}' | '\u{1680}' | '\u{2000}'
..='\u{200a}' | '\u{202f}' | '\u{205f}' | '\u{3000}' )
}
pub(crate) fn is_line_terminator(ch: char) -> bool {
matches!(ch, '\u{000a}' | '\u{000d}' | '\u{2028}' | '\u{2029}')
}
impl<'a> Lexer<'a> {
pub(super) fn skip_trivia(&mut self) -> Result<bool, LexError> {
let mut newline = false;
loop {
match self.cursor.peek() {
Some(ch) if is_line_terminator(ch) => {
newline = true;
let _ = self.cursor.bump();
}
Some(ch) if is_whitespace(ch) => {
let _ = self.cursor.bump();
}
Some('/') => match self.cursor.peek_byte(1) {
Some(b'/') => self.skip_line_comment(),
Some(b'*') => newline |= self.skip_block_comment()?,
_ => return Ok(newline),
},
Some('#')
if self.cursor.offset() == 0 && self.cursor.peek_byte(1) == Some(b'!') =>
{
self.skip_line_comment();
}
_ => return Ok(newline),
}
}
}
fn skip_line_comment(&mut self) {
self.cursor.advance_ascii(2);
while let Some(ch) = self.cursor.peek() {
if is_line_terminator(ch) {
return;
}
let _ = self.cursor.bump();
}
}
fn skip_block_comment(&mut self) -> Result<bool, LexError> {
let start = self.cursor.offset();
self.cursor.advance_ascii(2);
let mut newline = false;
loop {
if self.cursor.starts_with("*/") {
self.cursor.advance_ascii(2);
return Ok(newline);
}
match self.cursor.bump() {
Some(ch) => newline |= is_line_terminator(ch),
None => {
return Err(LexError {
kind: LexErrorKind::UnterminatedComment,
span: Span::new(start, self.cursor.offset()),
});
}
}
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::lexer::Goal;
use crate::lexer::TokenKind;
use crate::lexer::test_support::*;
#[test]
fn every_ecmascript_line_terminator_sets_newline_before() {
for terminator in ["\n", "\r", "\u{2028}", "\u{2029}"] {
let source = format!("{terminator};");
assert!(
first(&source).newline_before,
"{terminator:?} should end a line"
);
}
let token = first("\r\n;");
assert!(token.newline_before);
assert_eq!(token.span, Span::new(2, 3));
}
#[test]
fn plain_white_space_does_not_set_newline_before() {
for space in [" ", "\t", "\u{000b}", "\u{000c}", "\u{00a0}", "\u{feff}"] {
let source = format!("{space};");
assert!(
!first(&source).newline_before,
"{space:?} is white space, not a line terminator"
);
}
assert!(!first(";").newline_before);
}
#[test]
fn the_white_space_set_is_the_spec_table_not_rusts_idea_of_white_space() {
assert!(!'\u{feff}'.is_whitespace());
assert_eq!(kinds("\u{feff};"), [TokenKind::Semicolon, TokenKind::Eof]);
assert!('\u{0085}'.is_whitespace());
assert_eq!(
Lexer::new("\u{0085}").next_token(Goal::Div).map(|t| t.kind),
Err(LexError {
kind: LexErrorKind::UnexpectedCharacter,
span: Span::new(0, 2),
})
);
assert!(is_space_separator('\u{200a}')); assert!(!is_space_separator('\u{200b}')); assert!(!is_space_separator('\u{1fff}')); for space in [
'\u{0020}', '\u{00a0}', '\u{1680}', '\u{2000}', '\u{2005}', '\u{202f}', '\u{205f}',
'\u{3000}',
] {
assert!(is_space_separator(space), "{space:?} is in Zs");
assert!(is_whitespace(space), "{space:?} is <USP>");
}
for space in ['\u{0009}', '\u{000b}', '\u{000c}', '\u{feff}'] {
assert!(is_whitespace(space), "{space:?} is in Table 31");
assert!(!is_space_separator(space), "{space:?} is not Zs");
}
for terminator in ['\n', '\r', '\u{2028}', '\u{2029}'] {
assert!(is_line_terminator(terminator));
assert!(!is_whitespace(terminator));
}
assert!(!is_line_terminator('\u{2027}')); assert!(!is_line_terminator('\u{202a}')); assert!(!is_line_terminator(' '));
}
#[test]
fn a_line_comment_stops_before_the_terminator_that_still_ends_the_line() {
let token = first("//comment\n;");
assert_eq!(token.kind, TokenKind::Semicolon);
assert!(token.newline_before);
assert_eq!(kinds("//;;;"), [TokenKind::Eof]);
assert!(!first("//comment").newline_before);
assert!(first("//comment\u{2028};").newline_before);
assert_eq!(kinds("/"), [TokenKind::Slash, TokenKind::Eof]);
assert_eq!(
kinds("/=;"),
[TokenKind::SlashEq, TokenKind::Semicolon, TokenKind::Eof]
);
assert_eq!(kinds("///x"), [TokenKind::Eof]);
}
#[test]
fn a_block_comment_spanning_lines_counts_as_a_line_terminator() {
assert!(first("/*\n*/;").newline_before);
assert!(first("/*\r*/;").newline_before);
assert!(first("/*\u{2028}*/;").newline_before);
assert!(first("/*\u{2029}*/;").newline_before);
assert!(!first("/* no break here */;").newline_before);
assert!(first("/*\n*/ /* and more */ ;").newline_before);
assert!(first("\n/* x */;").newline_before);
}
#[test]
fn block_comments_end_at_the_first_close_and_do_not_nest() {
assert_eq!(kinds("/* /* */;"), [TokenKind::Semicolon, TokenKind::Eof]);
assert_eq!(kinds("/***/;"), [TokenKind::Semicolon, TokenKind::Eof]);
assert_eq!(kinds("/* * */;"), [TokenKind::Semicolon, TokenKind::Eof]);
assert_eq!(kinds("/**/;"), [TokenKind::Semicolon, TokenKind::Eof]);
assert_eq!(kinds("/*/*/;"), [TokenKind::Semicolon, TokenKind::Eof]);
assert_eq!(kinds("/* 🚀 é */;"), [TokenKind::Semicolon, TokenKind::Eof]);
}
#[test]
fn an_unterminated_block_comment_is_an_error_spanning_to_the_end_of_the_source() {
for source in ["/*", "/*/", "/* x", "/**", ";/* x\ny"] {
let start = source.find("/*").unwrap_or(0) as u32; assert_eq!(
Lexer::new(source).tokens(Goal::Div),
Err(LexError {
kind: LexErrorKind::UnterminatedComment,
span: Span::new(start, source.len() as u32),
}),
"on {source:?}"
);
}
assert_eq!(kinds("/*/ */;"), [TokenKind::Semicolon, TokenKind::Eof]);
}
#[test]
fn a_hashbang_is_a_comment_only_at_the_very_first_byte() {
assert_eq!(
kinds("#!/usr/bin/env node\n;"),
[TokenKind::Semicolon, TokenKind::Eof]
);
assert!(first("#!/usr/bin/env node\n;").newline_before);
assert_eq!(kinds("#!x"), [TokenKind::Eof]);
assert_eq!(
kinds("#!"),
[TokenKind::Eof],
"the shortest hashbang there is"
);
assert_eq!(
Lexer::new("#!x\n#!y").tokens(Goal::Div),
Err(LexError {
kind: LexErrorKind::UnexpectedCharacter,
span: Span::new(5, 6),
})
);
for source in [" #!x", "\n#!x", ";#!x"] {
assert_eq!(
Lexer::new(source).tokens(Goal::Div).map(|t| t.len()),
Err(LexError {
kind: LexErrorKind::UnexpectedCharacter,
span: Span::new(2, 3),
}),
"on {source:?}"
);
}
assert_eq!(
kinds("#x"),
[
TokenKind::PrivateIdentifier {
contains_escape: false
},
TokenKind::Eof
]
);
}
}