use super::{LexError, LexErrorKind, Lexer, ReservedWord, TokenKind};
use crate::span::Span;
use crate::unicode_id::{is_id_continue, is_id_start};
use std::borrow::Cow;
pub fn identifier_value<'a>(source: &'a str, span: Span) -> Option<Cow<'a, str>> {
let text = span.slice(source)?;
if !text.contains('\\') {
return Some(Cow::Borrowed(text));
}
let mut lexer = Lexer::new(text);
let mut value = String::with_capacity(text.len());
while !lexer.cursor.is_eof() {
if lexer.cursor.starts_with("\\") {
match lexer.read_unicode_escape() {
Ok(code_point) => value.push(char::from_u32(code_point)?),
Err(_) => return None,
}
} else {
value.push(lexer.cursor.bump()?);
}
}
Some(Cow::Owned(value))
}
impl<'a> Lexer<'a> {
pub(super) fn classify_name(&self, span: Span, contains_escape: bool) -> TokenKind {
if contains_escape {
return TokenKind::Identifier {
contains_escape: true,
};
}
match span
.slice(self.cursor.source)
.and_then(ReservedWord::from_text)
{
Some(word) => TokenKind::Keyword(word),
None => TokenKind::Identifier {
contains_escape: false,
},
}
}
pub(super) fn scan_identifier(&mut self) -> Result<bool, LexError> {
let at = self.cursor.offset();
let mut contains_escape = match self.cursor.peek() {
Some('\\') => self.scan_escaped_identifier_char(true)?,
Some(ch) if is_id_start(ch as u32) => {
let _ = self.cursor.bump();
false
}
_ => {
let _ = self.cursor.bump();
return Err(LexError {
kind: LexErrorKind::UnexpectedCharacter,
span: Span::new(at, self.cursor.offset()),
});
}
};
loop {
match self.cursor.peek() {
Some('\\') => contains_escape |= self.scan_escaped_identifier_char(false)?,
Some(ch) if is_id_continue(ch as u32) => {
let _ = self.cursor.bump();
}
_ => return Ok(contains_escape),
}
}
}
fn scan_escaped_identifier_char(&mut self, is_start: bool) -> Result<bool, LexError> {
let at = self.cursor.offset();
let code_point = self.read_unicode_escape()?;
let allowed = if is_start {
is_id_start(code_point)
} else {
is_id_continue(code_point)
};
if !allowed {
return Err(LexError {
kind: LexErrorKind::EscapedCodePointIsNotAnIdentifierCharacter,
span: Span::new(at, self.cursor.offset()),
});
}
Ok(true)
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::lexer::Goal;
use crate::lexer::test_support::*;
fn name_of(source: &str) -> String {
let token = first(source);
identifier_value(source, token.span)
.unwrap_or_else(|| panic!("{source:?} should have an identifier value")) .into_owned()
}
#[test]
fn a_name_runs_to_the_first_character_that_cannot_continue_it() {
assert_eq!(kinds("a"), [PLAIN, TokenKind::Eof]);
assert_eq!(kinds("abc"), [PLAIN, TokenKind::Eof]);
assert_eq!(first("abc;").span, Span::new(0, 3));
assert_eq!(kinds("a b"), [PLAIN, PLAIN, TokenKind::Eof]);
assert_eq!(
kinds("a-b"),
[PLAIN, TokenKind::Minus, PLAIN, TokenKind::Eof]
);
assert_eq!(kinds("_"), [PLAIN, TokenKind::Eof]);
assert_eq!(kinds("$"), [PLAIN, TokenKind::Eof]);
assert_eq!(kinds("$_0"), [PLAIN, TokenKind::Eof]);
assert_eq!(first("a0b$_;").span, Span::new(0, 5));
assert_eq!(
Lexer::new("1").next_token(Goal::Div).map(|t| t.kind),
Ok(TokenKind::Number { legacy: false })
);
assert_eq!(
Lexer::new("1abc").tokens(Goal::Div).map(|t| t.len()),
Err(LexError {
kind: LexErrorKind::NumericLiteralFollowedByIdentifierOrDigit,
span: Span::new(1, 2),
})
);
assert_eq!(
kinds("a=>b"),
[PLAIN, TokenKind::Arrow, PLAIN, TokenKind::Eof]
);
}
#[test]
fn names_use_the_unicode_id_sets_and_not_an_ascii_approximation_of_them() {
for source in [
"caf\u{e9}",
"\u{3a9}mega",
"\u{5d0}",
"\u{3042}",
"\u{4e00}",
] {
assert_eq!(kinds(source), [PLAIN, TokenKind::Eof], "lexing {source:?}");
assert_eq!(first(source).span, Span::new(0, source.len() as u32));
}
assert_eq!(kinds("x\u{1d49c}"), [PLAIN, TokenKind::Eof]);
assert_eq!(first("x\u{1d49c}").span, Span::new(0, 5));
assert_eq!(kinds("\u{2118}"), [PLAIN, TokenKind::Eof]);
assert_eq!(kinds("x\u{b7}y"), [PLAIN, TokenKind::Eof]);
assert_eq!(kinds("x\u{200c}y"), [PLAIN, TokenKind::Eof]);
assert!(Lexer::new("x\u{200b}y").tokens(Goal::Div).is_err());
for source in ["\u{20ac}", "\u{1f680}", "\u{00a7}"] {
assert!(Lexer::new(source).tokens(Goal::Div).is_err(), "{source:?}");
}
}
#[test]
fn a_reserved_word_spelled_with_an_escape_is_a_name_and_not_a_keyword() {
assert_eq!(kinds("els\\u{65}"), [ESCAPED, TokenKind::Eof]);
assert_eq!(name_of("els\\u{65}"), "else");
assert_eq!(kinds("\\u0069f"), [ESCAPED, TokenKind::Eof]);
assert_eq!(name_of("\\u0069f"), "if");
assert_eq!(
kinds("else"),
[TokenKind::Keyword(ReservedWord::Else), TokenKind::Eof]
);
assert_eq!(
kinds("if"),
[TokenKind::Keyword(ReservedWord::If), TokenKind::Eof]
);
}
#[test]
fn a_unicode_escape_contributes_a_code_point_to_the_name() {
assert_eq!(name_of("\\u0061bc"), "abc");
assert_eq!(name_of("a\\u{62}c"), "abc");
assert_eq!(name_of("\\u{61}\\u{62}"), "ab");
assert_eq!(kinds("\\u0061bc"), [ESCAPED, TokenKind::Eof]);
assert_eq!(name_of("\\u00E9"), "\u{e9}");
assert_eq!(name_of("\\u00e9"), "\u{e9}");
assert_eq!(name_of("\\u{E9}"), "\u{e9}");
assert_eq!(name_of("\\u{000000000000061}"), "a");
assert_eq!(name_of("\\u{1D49C}"), "\u{1d49c}");
assert_eq!(first("\\u{61}").span, Span::new(0, 6));
assert_eq!(name_of("u0061"), "u0061");
}
#[test]
fn an_escape_must_denote_a_character_that_could_have_been_written_directly() {
let not_a_name_char = |source: &str| {
assert_eq!(
Lexer::new(source).tokens(Goal::Div).map(|t| t.len()),
Err(LexError {
kind: LexErrorKind::EscapedCodePointIsNotAnIdentifierCharacter,
span: Span::new(0, source.len() as u32),
}),
"on {source:?}"
);
};
not_a_name_char("\\u0020"); not_a_name_char("\\u{2e}"); not_a_name_char("\\uD800"); not_a_name_char("\\u{10FFFF}");
assert_eq!(
Lexer::new("\\u0030").tokens(Goal::Div),
Err(LexError {
kind: LexErrorKind::EscapedCodePointIsNotAnIdentifierCharacter,
span: Span::new(0, 6),
})
);
assert_eq!(name_of("a\\u0030"), "a0");
assert_eq!(name_of("\\u005F"), "_");
assert_eq!(name_of("a\\u005F"), "a_");
assert_eq!(
Lexer::new("ab\\u0020").tokens(Goal::Div),
Err(LexError {
kind: LexErrorKind::EscapedCodePointIsNotAnIdentifierCharacter,
span: Span::new(2, 8),
})
);
}
#[test]
fn a_malformed_escape_is_an_error_rather_than_a_shorter_name() {
for (source, start, end) in [
("\\", 0, 1), ("\\x", 0, 1), ("\\u", 0, 2), ("\\u12", 0, 4), ("\\u123", 0, 5), ("\\u12g4", 0, 4), ("\\u{", 0, 3), ("\\u{}", 0, 3), ("\\u{61", 0, 5), ("\\u{zz}", 0, 3), ("\\u{61 }", 0, 5), ("\\u{6_1}", 0, 4), ("a\\x", 1, 2), ("a\\u{}", 1, 4), ] {
assert_eq!(
Lexer::new(source).tokens(Goal::Div).map(|t| t.len()),
Err(LexError {
kind: LexErrorKind::InvalidUnicodeEscape,
span: Span::new(start, end),
}),
"on {source:?}"
);
}
assert_eq!(name_of("\\u00610"), "a0");
assert_eq!(name_of("\\u0061a"), "aa");
}
#[test]
fn an_escape_beyond_the_last_code_point_is_out_of_range_rather_than_malformed() {
for source in [
"\\u{110000}", "\\u{FFFFFFFF}", "\\u{FFFFFFFFFFFFFF}", ] {
assert_eq!(
Lexer::new(source).tokens(Goal::Div).map(|t| t.len()),
Err(LexError {
kind: LexErrorKind::CodePointOutOfRange,
span: Span::new(0, source.len() as u32),
}),
"on {source:?}"
);
}
assert_eq!(
Lexer::new("\\u{10FFFF}").tokens(Goal::Div),
Err(LexError {
kind: LexErrorKind::EscapedCodePointIsNotAnIdentifierCharacter,
span: Span::new(0, 10),
})
);
}
#[test]
fn a_private_name_keeps_its_hash_in_both_the_span_and_the_value() {
let private = TokenKind::PrivateIdentifier {
contains_escape: false,
};
assert_eq!(kinds("#x"), [private, TokenKind::Eof]);
assert_eq!(first("#x").span, Span::new(0, 2));
assert_eq!(name_of("#x"), "#x");
assert_eq!(name_of("#count"), "#count");
assert_eq!(
kinds("this.#x"),
[
TokenKind::Keyword(ReservedWord::This),
TokenKind::Dot,
private,
TokenKind::Eof
]
);
assert_eq!(
kinds("#\\u0078"),
[
TokenKind::PrivateIdentifier {
contains_escape: true
},
TokenKind::Eof
]
);
assert_eq!(name_of("#\\u0078"), "#x");
for (source, span) in [
("#", Span::new(1, 1)),
("#5", Span::new(1, 2)),
("# x", Span::new(1, 2)),
("#\u{200b}", Span::new(1, 4)),
] {
assert_eq!(
Lexer::new(source).tokens(Goal::Div).map(|t| t.len()),
Err(LexError {
kind: LexErrorKind::UnexpectedCharacter,
span,
}),
"on {source:?}"
);
}
}
#[test]
fn identifier_value_borrows_the_common_case_and_owns_only_what_it_must() {
let source = "plain \\u0061bc";
let plain = first(source);
assert!(matches!(
identifier_value(source, plain.span),
Some(Cow::Borrowed("plain"))
));
let escaped = Lexer::new(source)
.tokens(Goal::Div)
.expect("this lexes") [1];
assert!(matches!(
identifier_value(source, escaped.span),
Some(Cow::Owned(ref value)) if value == "abc"
));
assert_eq!(identifier_value("abc", Span::new(0, 99)), None);
assert_eq!(identifier_value("\u{e9}", Span::new(0, 1)), None);
assert_eq!(identifier_value("a\\u", Span::new(0, 3)), None);
assert_eq!(
identifier_value("a\\u{20}", Span::new(0, 7)).as_deref(),
Some("a ")
);
assert!(
Lexer::new("a\\u{20}").tokens(Goal::Div).is_err(),
"…and the lexer does refuse it"
);
assert_eq!(
identifier_value("abc", Span::empty_at(1)).as_deref(),
Some("")
);
}
}