use winnow::{
ascii::space0,
combinator::{alt, dispatch, eof, fail, opt, peek, preceded},
token::{any, take},
};
use super::{
identifier::{identifier, is_identifier_special, is_identifier_start},
numeric::{number, ordinal},
prelude::*,
string,
};
pub(super) fn token<'s>(
input: &mut Input<'s>,
prev_token: Option<&mut Token<'s>>,
) -> Result<Token<'s>> {
let dot = prev_token.as_deref().is_some_and(|t| *t == Operator::Dot);
let token_parser = |i: &mut Input<'s>| {
dispatch! {peek(any);
'0'..='9' => if dot {
ordinal
} else {
number
},
'@' => alt((
string::string,
identifier(!dot),
)),
'"' | '\'' | '`' => string::string,
'+' => alt((
"+=".value(TokenKind::Operator(Operator::PlusAssign)),
any.value(TokenKind::Operator(Operator::Plus)),
)),
'-' => alt((
"-=".value(TokenKind::Operator(Operator::MinusAssign)),
"->".value(TokenKind::Operator(Operator::Arrow)),
any.value(TokenKind::Operator(Operator::Minus)),
)),
'*' => alt((
"*=".value(TokenKind::Operator(Operator::AsteriskAssign)),
any.value(TokenKind::Operator(Operator::Asterisk)),
)),
'/' => dispatch! {peek(opt(take(2usize)));
Some("/=") => take(2usize).value(TokenKind::Operator(Operator::SlashAssign)),
_ => any.value(TokenKind::Operator(Operator::Slash)),
},
'%' => alt((
"%=".value(TokenKind::Operator(Operator::PercentAssign)),
any.value(TokenKind::Operator(Operator::Percent)),
)),
'^' => alt((
"^=".value(TokenKind::Operator(Operator::CaretAssign)),
any.value(TokenKind::Operator(Operator::Caret)),
)),
'=' => alt((
"==".value(TokenKind::Operator(Operator::Equal)),
"=~".value(TokenKind::Operator(Operator::TildeEqual)),
any.value(TokenKind::Operator(Operator::Assign)),
)),
'!' => alt((
("!", peek(alt(("==", "=~")))).value(TokenKind::Operator(Operator::Exclamation)),
"!=".value(TokenKind::Operator(Operator::NotEqual)),
"!~".value(TokenKind::Operator(Operator::TildeNotEqual)),
any.value(TokenKind::Operator(Operator::Exclamation)),
)),
'>' => alt((
">=".value(TokenKind::Operator(Operator::GreaterEqual)),
">".value(TokenKind::Operator(Operator::Greater)),
)),
'<' => alt((
"<=".value(TokenKind::Operator(Operator::LessEqual)),
"<".value(TokenKind::Operator(Operator::Less)),
)),
'&' => alt((
"&&=".value(TokenKind::Operator(Operator::LogicalAndAssign)),
"&&".value(TokenKind::Operator(Operator::LogicalAnd)),
)),
'|' => alt((
"||=".value(TokenKind::Operator(Operator::LogicalOrAssign)),
"||".value(TokenKind::Operator(Operator::LogicalOr)),
)),
'?' => alt((
"?:".value(TokenKind::Operator(Operator::QuestionColon)),
"??=".value(TokenKind::Operator(Operator::NullCoalescingAssign)),
"??".value(TokenKind::Operator(Operator::NullCoalescing)),
any.value(TokenKind::Operator(Operator::Question)),
)),
'(' => any.value(TokenKind::Operator(Operator::OpenParen)),
')' => any.value(TokenKind::Operator(Operator::CloseParen)),
'[' => any.value(TokenKind::Operator(Operator::OpenBracket)),
']' => any.value(TokenKind::Operator(Operator::CloseBracket)),
':' => alt((
"::".value(TokenKind::Operator(Operator::ColonColon)),
any.value(TokenKind::Operator(Operator::Colon)),
)),
',' => any.value(TokenKind::Operator(Operator::Comma)),
'.' => alt((
"..<".value(TokenKind::Operator(Operator::HalfOpenRange)),
"..".value(TokenKind::Operator(Operator::SpreadRange)),
".".value(TokenKind::Operator(Operator::Dot)),
)),
';' => any.value(TokenKind::Operator(Operator::Semicolon)),
'{' => any.value(TokenKind::Operator(Operator::OpenBrace)),
'}' => any.value(TokenKind::Operator(Operator::CloseBrace)),
c if is_identifier_start(c) || is_identifier_special(c) => identifier(!dot),
_ => fail,
}
.parse_next(i)
};
let parse_token = |i: &mut Input<'s>| {
preceded(
space0,
alt((
token_parser,
eof.map(|_| TokenKind::Eof),
any.span().map(|range| {
TokenKind::unknown(TokenKind::Empty, range, DiagnosticCode::UnknownToken)
}),
))
.with_span()
.map(|(kind, range)| Token::new(kind, range)),
)
.parse_next(i)
};
let cur_token = parse_token(input)?;
if let Some(prev_token) = prev_token
&& let TokenKind::Keyword(prev_kw) = prev_token.kind
{
if cur_token == Operator::Colon || cur_token == Operator::QuestionColon {
prev_token.kind = TokenKind::Identifier(prev_kw.into());
} else if prev_kw == Keyword::Type
&& !(cur_token == Operator::OpenParen || cur_token.is_identifier())
{
prev_token.kind = TokenKind::Identifier(prev_kw.into());
} else if cur_token == Keyword::In && prev_kw == Keyword::Not {
prev_token.kind = TokenKind::Keyword(Keyword::NotIn);
prev_token.range.end = cur_token.range.end;
return Ok(Token::empty(cur_token.range.end));
}
}
Ok(cur_token)
}