#[cfg(test)]
mod tests;
use crate::syntax::ast::{
punc::Punctuator,
token::{Token, TokenKind},
};
use std::{
char::{decode_utf16, from_u32},
error, fmt,
iter::Peekable,
str::{Chars, FromStr},
};
macro_rules! vop {
($this:ident, $assign_op:expr, $op:expr) => ({
let preview = $this.preview_next().ok_or_else(|| LexerError::new("Could not preview next value"))?;
match preview {
'=' => {
$this.next();
$this.column_number += 1;
$assign_op
}
_ => $op,
}
});
($this:ident, $assign_op:expr, $op:expr, {$($case:pat => $block:expr), +}) => ({
let preview = $this.preview_next().ok_or_else(|| LexerError::new("Could not preview next value"))?;
match preview {
'=' => {
$this.next();
$this.column_number += 1;
$assign_op
},
$($case => {
$this.next();
$this.column_number += 1;
$block
})+,
_ => $op
}
});
($this:ident, $op:expr, {$($case:pat => $block:expr),+}) => {
let preview = $this.preview_next().ok_or_else(|| LexerError::new("Could not preview next value"))?;
match preview {
$($case => {
$this.next()?;
$this.column_number += 1;
$block
})+,
_ => $op
}
}
}
macro_rules! op {
($this:ident, $assign_op:expr, $op:expr) => ({
let punc = vop!($this, $assign_op, $op);
$this.push_punc(punc);
});
($this:ident, $assign_op:expr, $op:expr, {$($case:pat => $block:expr),+}) => ({
let punc = vop!($this, $assign_op, $op, {$($case => $block),+});
$this.push_punc(punc);
});
($this:ident, $op:expr, {$($case:pat => $block:expr),+}) => ({
let punc = vop!($this, $op, {$($case => $block),+});
$this.push_punc();
});
}
#[derive(Debug, Clone)]
pub struct LexerError {
details: String,
}
impl LexerError {
fn new(msg: &str) -> Self {
Self {
details: msg.to_string(),
}
}
}
impl fmt::Display for LexerError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "{}", self.details)
}
}
impl error::Error for LexerError {
fn description(&self) -> &str {
&self.details
}
fn cause(&self) -> Option<&dyn error::Error> {
None
}
}
#[derive(Debug)]
pub struct Lexer<'a> {
pub tokens: Vec<Token>,
line_number: u64,
column_number: u64,
buffer: Peekable<Chars<'a>>,
}
impl<'a> Lexer<'a> {
pub fn new(buffer: &'a str) -> Lexer<'a> {
Lexer {
tokens: Vec::new(),
line_number: 1,
column_number: 0,
buffer: buffer.chars().peekable(),
}
}
fn push_token(&mut self, tk: TokenKind) {
self.tokens
.push(Token::new(tk, self.line_number, self.column_number))
}
fn push_punc(&mut self, punc: Punctuator) {
self.push_token(TokenKind::Punctuator(punc));
}
fn next(&mut self) -> char {
self.buffer.next().expect(
"No more more characters to consume from input stream, \
use preview_next() first to check before calling next()",
)
}
fn preview_next(&mut self) -> Option<char> {
self.buffer.peek().copied()
}
fn preview_multiple_next(&mut self, nb_next: usize) -> Option<char> {
let mut next_peek = None;
for (i, x) in self.buffer.clone().enumerate() {
if i >= nb_next {
break;
}
next_peek = Some(x);
}
next_peek
}
fn take_char_while<F>(&mut self, mut f: F) -> Result<String, LexerError>
where
F: FnMut(char) -> bool,
{
let mut s = String::new();
while self.buffer.peek().is_some()
&& f(self.preview_next().expect("Could not preview next value"))
{
s.push(self.next());
}
Ok(s)
}
fn next_is(&mut self, peek: char) -> bool {
let result = self.preview_next() == Some(peek);
if result {
self.buffer.next();
}
result
}
fn read_integer_in_base(&mut self, base: u32, mut buf: String) -> Result<u64, LexerError> {
self.next();
while let Some(ch) = self.preview_next() {
if ch.is_digit(base) {
buf.push(self.next());
} else {
break;
}
}
u64::from_str_radix(&buf, base)
.map_err(|_| LexerError::new("Could not convert value to u64"))
}
fn check_after_numeric_literal(&mut self) -> Result<(), LexerError> {
match self.preview_next() {
Some(ch)
if ch.is_ascii_alphabetic() || ch == '$' || ch == '_' || ch.is_ascii_digit() =>
{
Err(LexerError::new("NumericLiteral token must not be followed by IdentifierStart nor DecimalDigit characters"))
}
Some(_) => Ok(()),
None => Ok(())
}
}
pub fn lex(&mut self) -> Result<(), LexerError> {
loop {
if self.preview_next().is_none() {
return Ok(());
}
self.column_number += 1;
let ch = self.next();
match ch {
'"' | '\'' => {
let mut buf = String::new();
loop {
if self.preview_next().is_none() {
return Err(LexerError::new("Unterminated String"));
}
match self.next() {
'\'' if ch == '\'' => {
break;
}
'"' if ch == '"' => {
break;
}
'\\' => {
if self.preview_next().is_none() {
return Err(LexerError::new("Unterminated String"));
}
let escape = self.next();
if escape != '\n' {
let escaped_ch = match escape {
'n' => '\n',
'r' => '\r',
't' => '\t',
'b' => '\x08',
'f' => '\x0c',
'0' => '\0',
'x' => {
let mut nums = String::with_capacity(2);
for _ in 0_u8..2 {
if self.preview_next().is_none() {
return Err(LexerError::new("Unterminated String"));
}
nums.push(self.next());
}
self.column_number += 2;
let as_num = match u64::from_str_radix(&nums, 16) {
Ok(v) => v,
Err(_) => 0,
};
match from_u32(as_num as u32) {
Some(v) => v,
None => panic!(
"{}:{}: {} is not a valid unicode scalar value",
self.line_number, self.column_number, as_num
),
}
}
'u' => {
if self.next_is('{') {
let s = self
.take_char_while(char::is_alphanumeric)
.expect("Could not read chars");
let as_num = match u32::from_str_radix(&s, 16) {
Ok(v) => v,
Err(_) => 0,
};
let c = from_u32(as_num).ok_or_else(|| LexerError::new("Invalid Unicode escape sequence"))?;
if self.preview_next().is_none() {
return Err(LexerError::new("Unterminated String"));
}
self.next();
self.column_number +=
(s.len() as u64).wrapping_add(3);
c
} else {
let mut codepoints: Vec<u16> = vec![];
loop {
let s = self
.take_char_while(char::is_alphanumeric)
.expect("Could not read chars");
let as_num = match u16::from_str_radix(&s, 16) {
Ok(v) => v,
Err(_) => 0,
};
codepoints.push(as_num);
self.column_number +=
(s.len() as u64).wrapping_add(2);
if self.next_is('\\') && self.next_is('u') {
continue;
}
break;
}
decode_utf16(codepoints.iter().cloned())
.next()
.expect("Could not get next codepoint")
.expect("Could not get next codepoint")
}
}
'\'' | '"' | '\\' => escape,
ch => {
let details = format!("{}:{}: Invalid escape `{}`", self.line_number, self.column_number, ch);
return Err(LexerError { details });
}
};
buf.push(escaped_ch);
}
}
next_ch => buf.push(next_ch),
}
}
let str_length = buf.len() as u64;
self.push_token(TokenKind::StringLiteral(buf));
self.column_number += str_length.wrapping_add(1);
}
'0' => {
let mut buf = String::new();
let num = match self.preview_next() {
None => {
self.push_token(TokenKind::NumericLiteral(0_f64));
return Ok(());
}
Some('x') | Some('X') => {
self.read_integer_in_base(16, buf)? as f64
}
Some('o') | Some('O') => {
self.read_integer_in_base(8, buf)? as f64
}
Some('b') | Some('B') => {
self.read_integer_in_base(2, buf)? as f64
}
Some(ch) if (ch.is_ascii_digit() || ch == '.') => {
let mut gone_decimal = ch == '.';
while let Some(next_ch) = self.preview_next() {
match next_ch {
c if next_ch.is_digit(8) => {
buf.push(c);
self.next();
}
'8' | '9' | '.' => {
gone_decimal = true;
buf.push(next_ch);
self.next();
}
_ => {
break;
}
}
}
if gone_decimal {
f64::from_str(&buf).map_err(|_e| LexerError::new("Could not convert value to f64"))?
} else if buf.is_empty() {
0.0
} else {
(u64::from_str_radix(&buf, 8).map_err(|_e| LexerError::new("Could not convert value to u64"))?) as f64
}
}
Some(_) => {
0.0
}
};
self.push_token(TokenKind::NumericLiteral(num));
if let Err(e) = self.check_after_numeric_literal() {
return Err(e)
};
}
_ if ch.is_digit(10) => {
let mut buf = ch.to_string();
'digitloop: while let Some(ch) = self.preview_next() {
match ch {
'.' => loop {
buf.push(self.next());
let c = match self.preview_next() {
Some(ch) => ch,
None => break,
};
match c {
'e' | 'E' => {
match self.preview_multiple_next(2).unwrap_or_default().to_digit(10) {
Some(0..=9) | None => {
buf.push(self.next());
}
_ => {
break 'digitloop;
}
}
}
_ => {
if !c.is_digit(10) {
break 'digitloop;
}
}
}
},
'e' | 'E' => {
match self.preview_multiple_next(2).unwrap_or_default().to_digit(10) {
Some(0..=9) | None => {
buf.push(self.next());
}
_ => {
break;
}
}
buf.push(self.next());
}
'+' | '-' => {
break;
}
_ if ch.is_digit(10) => {
buf.push(self.next());
}
_ => break,
}
}
self.push_token(TokenKind::NumericLiteral(
f64::from_str(&buf).map_err(|_| LexerError::new("Could not convert value to f64"))?,
))
}
_ if ch.is_alphabetic() || ch == '$' || ch == '_' => {
let mut buf = ch.to_string();
while let Some(ch) = self.preview_next() {
if ch.is_alphabetic() || ch.is_digit(10) || ch == '_' {
buf.push(self.next());
} else {
break;
}
}
let buf_compare: &str = &buf;
self.push_token(match buf_compare {
"true" => TokenKind::BooleanLiteral(true),
"false" => TokenKind::BooleanLiteral(false),
"null" => TokenKind::NullLiteral,
slice => {
if let Ok(keyword) = FromStr::from_str(slice) {
TokenKind::Keyword(keyword)
} else {
TokenKind::Identifier(buf.clone())
}
}
});
self.column_number += (buf_compare.len().wrapping_sub(1)) as u64;
}
';' => self.push_punc(Punctuator::Semicolon),
':' => self.push_punc(Punctuator::Colon),
'.' => {
if self.next_is('.') {
if self.next_is('.') {
self.push_punc(Punctuator::Spread);
self.column_number += 2;
} else {
return Err(LexerError::new("Expecting Token ."));
}
} else {
self.push_punc(Punctuator::Dot);
};
}
'(' => self.push_punc(Punctuator::OpenParen),
')' => self.push_punc(Punctuator::CloseParen),
',' => self.push_punc(Punctuator::Comma),
'{' => self.push_punc(Punctuator::OpenBlock),
'}' => self.push_punc(Punctuator::CloseBlock),
'[' => self.push_punc(Punctuator::OpenBracket),
']' => self.push_punc(Punctuator::CloseBracket),
'?' => self.push_punc(Punctuator::Question),
'/' => {
if let Some(ch) = self.preview_next() {
match ch {
'/' => {
while self.preview_next().is_some() {
if self.next() == '\n' {
break;
}
}
self.line_number += 1;
self.column_number = 0;
}
'*' => {
let mut lines = 0;
loop {
if self.preview_next().is_none() {
return Err(LexerError::new("Unterminated Multiline Comment"));
}
match self.next() {
'*' => {
if self.next_is('/') {
break;
}
}
next_ch => {
if next_ch == '\n' {
lines += 1;
}
},
}
}
self.line_number += lines;
self.column_number = 0;
}
_ => {
let original_buffer = self.buffer.clone();
let mut body = String::new();
let mut regex = false;
loop {
self.column_number +=1;
match self.buffer.next() {
Some('/') => {
regex = true;
break;
}
n @ Some('\n') | n @ Some('\r') | n @ Some('\u{2028}')
| n @ Some('\u{2029}') => {
self.column_number = 0;
if n != Some('\r') {
self.line_number += 1;
}
break
},
None => {
self.column_number -= 1;
break
}
Some('\\') => {
body.push('\\');
if self.preview_next().is_none() {
break;
}
match self.next() {
'\n' | '\r' | '\u{2028}' | '\u{2029}' => break,
ch => body.push(ch),
}
}
Some(ch) => body.push(ch),
}
}
if regex {
let flags = self.take_char_while(char::is_alphabetic)?;
self.push_token(TokenKind::RegularExpressionLiteral(
body, flags,
));
} else {
self.buffer = original_buffer;
if self.next_is('=') {
self.push_token(TokenKind::Punctuator(
Punctuator::AssignDiv,
));
} else {
self.push_token(TokenKind::Punctuator(Punctuator::Div));
}
}
}
}
} else {
return Err(LexerError::new("Expecting Token /,*,= or regex"));
}
}
'*' => op!(self, Punctuator::AssignMul, Punctuator::Mul, {
'*' => vop!(self, Punctuator::AssignPow, Punctuator::Exp)
}),
'+' => op!(self, Punctuator::AssignAdd, Punctuator::Add, {
'+' => Punctuator::Inc
}),
'-' => op!(self, Punctuator::AssignSub, Punctuator::Sub, {
'-' => {
Punctuator::Dec
}
}),
'%' => op!(self, Punctuator::AssignMod, Punctuator::Mod),
'|' => op!(self, Punctuator::AssignOr, Punctuator::Or, {
'|' => Punctuator::BoolOr
}),
'&' => op!(self, Punctuator::AssignAnd, Punctuator::And, {
'&' => Punctuator::BoolAnd
}),
'^' => op!(self, Punctuator::AssignXor, Punctuator::Xor),
'=' => op!(self, if self.next_is('=') {
Punctuator::StrictEq
} else {
Punctuator::Eq
}, Punctuator::Assign, {
'>' => {
Punctuator::Arrow
}
}),
'<' => op!(self, Punctuator::LessThanOrEq, Punctuator::LessThan, {
'<' => vop!(self, Punctuator::AssignLeftSh, Punctuator::LeftSh)
}),
'>' => op!(self, Punctuator::GreaterThanOrEq, Punctuator::GreaterThan, {
'>' => vop!(self, Punctuator::AssignRightSh, Punctuator::RightSh, {
'>' => vop!(self, Punctuator::AssignURightSh, Punctuator::URightSh)
})
}),
'!' => op!(
self,
vop!(self, Punctuator::StrictNotEq, Punctuator::NotEq),
Punctuator::Not
),
'~' => self.push_punc(Punctuator::Neg),
'\n' | '\u{2028}' | '\u{2029}' => {
self.push_token(TokenKind::LineTerminator);
self.line_number += 1;
self.column_number = 0;
}
'\r' => {
self.column_number = 0;
}
'\u{0020}' | '\u{0009}' | '\u{000B}' | '\u{000C}' | '\u{00A0}' | '\u{FEFF}' |
'\u{1680}' | '\u{2000}'..='\u{200A}' | '\u{202F}' | '\u{205F}' | '\u{3000}' => (),
_ => {
let details = format!("{}:{}: Unexpected '{}'", self.line_number, self.column_number, ch);
return Err(LexerError { details });
},
}
}
}
}