use super::unicode_ops::unicode_op_kind;
use crate::keywords::keyword_table;
use crate::tokens::token_table;
mod operators;
macro_rules! define_tok_kind {
($($(#[$meta:meta])* $tok:ident $syn:ident,)*) => {
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum TokKind {
$($(#[$meta])* $tok,)*
UniArrow,
UniComparison,
UniColon,
UniPlus,
UniTimes,
UniPower,
}
};
}
token_table!(define_tok_kind);
impl TokKind {
pub(crate) fn is_trivia(self) -> bool {
matches!(
self,
TokKind::Whitespace | TokKind::Newline | TokKind::Comment | TokKind::BlockComment
)
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) struct Token<'src> {
pub(crate) kind: TokKind,
pub(crate) text: &'src str,
pub(crate) start: usize,
pub(crate) end: usize,
}
pub(crate) fn lex(input: &str) -> Vec<Token<'_>> {
Lexer::new(input).run()
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum Mode {
Str {
triple: bool,
raw: bool,
prefixed: bool,
},
Cmd {
triple: bool,
raw: bool,
prefixed: bool,
},
Interp { depth: usize },
}
pub(super) struct Lexer<'a> {
input: &'a str,
pub(super) bytes: &'a [u8],
pub(super) pos: usize,
tokens: Vec<Token<'a>>,
mode_stack: Vec<Mode>,
}
impl<'a> Lexer<'a> {
fn new(input: &'a str) -> Self {
Self {
input,
bytes: input.as_bytes(),
pos: 0,
tokens: Vec::new(),
mode_stack: Vec::new(),
}
}
fn run(mut self) -> Vec<Token<'a>> {
while self.pos < self.bytes.len() {
self.next_token();
}
self.tokens
}
fn peek(&self, ahead: usize) -> Option<u8> {
self.bytes.get(self.pos + ahead).copied()
}
fn push(&mut self, kind: TokKind, start: usize, end: usize) {
self.tokens.push(Token {
kind,
text: &self.input[start..end],
start,
end,
});
}
fn push_op(&mut self, kind: TokKind, start: usize) {
if op_takes_suffix(kind) {
while self.pos < self.bytes.len() {
let c = self.char_at(self.pos);
if is_op_suffix_char(c) {
self.pos += c.len_utf8();
} else {
break;
}
}
}
self.push(kind, start, self.pos);
}
fn next_token(&mut self) {
if matches!(
self.mode_stack.last(),
Some(Mode::Str { .. } | Mode::Cmd { .. })
) {
self.lex_in_string_mode();
return;
}
let start = self.pos;
let b = self.bytes[self.pos];
if matches!(self.mode_stack.last(), Some(Mode::Interp { .. })) {
if b == b'(' {
self.pos += 1;
self.push(TokKind::LParen, start, self.pos);
if let Some(Mode::Interp { depth }) = self.mode_stack.last_mut() {
*depth += 1;
}
return;
}
if b == b')' {
self.pos += 1;
self.push(TokKind::RParen, start, self.pos);
if matches!(self.mode_stack.last(), Some(Mode::Interp { depth }) if *depth == 1) {
self.mode_stack.pop();
} else if let Some(Mode::Interp { depth }) = self.mode_stack.last_mut() {
*depth -= 1;
}
return;
}
}
match b {
b' ' | b'\t' => self.lex_whitespace(start),
b'\r' | b'\n' => self.lex_newline(start),
b'#' => self.lex_comment(start),
b'"' => self.lex_open_string(start, false),
b'`' => self.lex_open_cmd(start, false),
b'\'' if self.prev_ends_value() || self.prev_is_dot() => {
self.pos += 1;
self.push_op(TokKind::Transpose, start);
}
b'\'' => self.lex_char_literal(start),
b'0'..=b'9' => self.lex_number(start),
b'.' if self.peek(1).is_some_and(|c| c.is_ascii_digit()) => self.lex_number(start),
_ => {
if is_ident_start(self.char_at(self.pos)) {
self.lex_ident_or_keyword(start);
} else {
self.lex_operator_or_unknown(start);
}
}
}
}
pub(super) fn char_at(&self, at: usize) -> char {
self.input[at..].chars().next().unwrap_or('\0')
}
fn lex_whitespace(&mut self, start: usize) {
while matches!(self.peek(0), Some(b' ' | b'\t')) {
self.pos += 1;
}
self.push(TokKind::Whitespace, start, self.pos);
}
fn lex_newline(&mut self, start: usize) {
match self.peek(0) {
Some(b'\r') if self.peek(1) == Some(b'\n') => self.pos += 2,
_ => self.pos += 1,
}
self.push(TokKind::Newline, start, self.pos);
}
fn lex_comment(&mut self, start: usize) {
if self.peek(1) == Some(b'=') {
self.lex_block_comment(start);
return;
}
self.pos += 1;
while !matches!(self.peek(0), Some(b'\n' | b'\r') | None) {
self.pos += 1;
}
self.push(TokKind::Comment, start, self.pos);
}
fn lex_block_comment(&mut self, start: usize) {
self.pos += 2; let mut depth = 1usize;
while depth > 0 && self.pos < self.bytes.len() {
match (self.peek(0), self.peek(1)) {
(Some(b'#'), Some(b'=')) => {
self.pos += 2;
depth += 1;
}
(Some(b'='), Some(b'#')) => {
self.pos += 2;
depth -= 1;
}
_ => self.pos += 1,
}
}
self.push(TokKind::BlockComment, start, self.pos);
}
fn lex_open_string(&mut self, start: usize, prefixed: bool) {
let triple = self.peek(1) == Some(b'"') && self.peek(2) == Some(b'"');
self.pos += if triple { 3 } else { 1 };
self.push(TokKind::StringDelimOpen, start, self.pos);
self.mode_stack.push(Mode::Str {
triple,
raw: prefixed,
prefixed,
});
}
fn lex_open_cmd(&mut self, start: usize, prefixed: bool) {
let triple = self.peek(1) == Some(b'`') && self.peek(2) == Some(b'`');
self.pos += if triple { 3 } else { 1 };
self.push(TokKind::CmdDelimOpen, start, self.pos);
self.mode_stack.push(Mode::Cmd {
triple,
raw: prefixed,
prefixed,
});
}
fn lex_in_string_mode(&mut self) {
let frame = *self.mode_stack.last().expect("string mode frame");
let (quote, triple, raw, prefixed) = match frame {
Mode::Str {
triple,
raw,
prefixed,
} => (b'"', triple, raw, prefixed),
Mode::Cmd {
triple,
raw,
prefixed,
} => (b'`', triple, raw, prefixed),
Mode::Interp { .. } => unreachable!("lex_in_string_mode called in interp mode"),
};
let close_kind = if quote == b'"' {
TokKind::StringDelimClose
} else {
TokKind::CmdDelimClose
};
let start = self.pos;
if self.at_close_delim(quote, triple) {
self.pos += if triple { 3 } else { 1 };
self.push(close_kind, start, self.pos);
self.mode_stack.pop();
if prefixed {
self.lex_suffix();
}
return;
}
while self.pos < self.bytes.len() {
if raw && self.peek(0) == Some(b'\\') {
let mut run = 0;
while self.peek(run) == Some(b'\\') {
run += 1;
}
if self.peek(run) == Some(quote) && run % 2 == 1 {
self.pos += run + 1;
} else {
self.pos += run;
}
continue;
}
if self.at_close_delim(quote, triple) {
break;
}
if !raw && self.peek(0) == Some(b'$') && self.is_interp_start(1) {
break;
}
self.consume_body_byte(raw);
}
if self.pos > start {
self.push(TokKind::StringContent, start, self.pos);
}
if self.at_close_delim(quote, triple) {
let delim_start = self.pos;
self.pos += if triple { 3 } else { 1 };
self.push(close_kind, delim_start, self.pos);
self.mode_stack.pop();
if prefixed {
self.lex_suffix();
}
} else if !raw && self.peek(0) == Some(b'$') && self.is_interp_start(1) {
self.lex_interp_sigil();
} else {
self.mode_stack.pop();
}
}
fn at_close_delim(&self, quote: u8, triple: bool) -> bool {
if triple {
self.peek(0) == Some(quote)
&& self.peek(1) == Some(quote)
&& self.peek(2) == Some(quote)
} else {
self.peek(0) == Some(quote)
}
}
fn is_interp_start(&self, ahead: usize) -> bool {
match self.peek(ahead) {
Some(b'(') => true,
Some(_) => is_ident_start(self.char_at(self.pos + ahead)),
None => false,
}
}
fn lex_interp_sigil(&mut self) {
let dollar = self.pos;
self.pos += 1;
self.push(TokKind::Dollar, dollar, self.pos);
if self.peek(0) == Some(b'(') {
let paren = self.pos;
self.pos += 1;
self.push(TokKind::LParen, paren, self.pos);
self.mode_stack.push(Mode::Interp { depth: 1 });
} else {
self.lex_interp_ident();
}
}
fn consume_body_byte(&mut self, raw: bool) {
if !raw && self.peek(0) == Some(b'\\') && self.pos + 1 < self.bytes.len() {
if self.peek(1) == Some(b'\r') && self.peek(2) == Some(b'\n') {
self.pos += 3;
} else {
self.pos += 1 + self.char_at(self.pos + 1).len_utf8();
}
} else {
self.pos += self.char_at(self.pos).len_utf8();
}
}
fn lex_suffix(&mut self) {
let start = self.pos;
if matches!(self.peek(0), Some(c) if c.is_ascii_alphabetic()) {
self.pos += 1;
while matches!(self.peek(0), Some(c) if c.is_ascii_alphanumeric()) {
self.pos += 1;
}
}
if self.pos > start {
self.push(TokKind::StringSuffix, start, self.pos);
}
}
fn prev_ends_value(&self) -> bool {
matches!(
self.tokens.last().map(|t| t.kind),
Some(
TokKind::Ident
| TokKind::Integer
| TokKind::Float
| TokKind::Char
| TokKind::TrueKw
| TokKind::FalseKw
| TokKind::RParen
| TokKind::RBracket
| TokKind::RBrace
| TokKind::StringDelimClose
| TokKind::CmdDelimClose
| TokKind::StringSuffix
| TokKind::Transpose
)
)
}
fn prev_is_dot(&self) -> bool {
self.tokens
.iter()
.rev()
.find(|t| !matches!(t.kind, TokKind::Whitespace | TokKind::BlockComment))
.map(|t| t.kind)
== Some(TokKind::Dot)
}
fn lex_char_literal(&mut self, start: usize) {
let mut idx = self.pos + 1;
let mut found = false;
while idx < self.bytes.len() {
match self.bytes[idx] {
b'\'' => {
found = true;
break;
}
b'\\' => {
idx += 1;
if idx < self.bytes.len() {
idx += self.char_at(idx).len_utf8();
}
}
_ => idx += self.char_at(idx).len_utf8(),
}
}
self.pos = if found { idx + 1 } else { idx };
self.push(TokKind::Char, start, self.pos);
}
fn lex_number(&mut self, start: usize) {
if self.peek(0) == Some(b'0') {
match self.peek(1) {
Some(b'x') => {
self.pos += 2;
let int_digits = self.consume_digits(|c| c.is_ascii_hexdigit());
let mut had_dot = false;
let mut frac_digits = false;
if self.peek(0) == Some(b'.') && self.peek(1) != Some(b'.') {
had_dot = true;
self.pos += 1;
frac_digits = self.consume_digits(|c| c.is_ascii_hexdigit());
}
let mut had_exponent = false;
let mut exp_digits = false;
if matches!(self.peek(0), Some(b'p' | b'P')) {
had_exponent = true;
self.pos += 1;
if matches!(self.peek(0), Some(b'+' | b'-')) {
self.pos += 1;
}
exp_digits = self.consume_digits(|c| c.is_ascii_digit());
}
let has_mantissa = int_digits || frac_digits;
let kind = if had_exponent {
if has_mantissa && exp_digits {
TokKind::Float
} else {
TokKind::ErrorInvalidNumber
}
} else if had_dot {
TokKind::ErrorHexFloatNoP
} else if int_digits {
TokKind::HexInt
} else {
TokKind::ErrorInvalidNumber
};
self.push(kind, start, self.pos);
return;
}
Some(b'o') => {
self.pos += 2;
self.consume_digits(|c| (b'0'..=b'7').contains(&c));
self.push(TokKind::OctInt, start, self.pos);
return;
}
Some(b'b') => {
self.pos += 2;
self.consume_digits(|c| matches!(c, b'0' | b'1'));
self.push(TokKind::BinInt, start, self.pos);
return;
}
_ => {}
}
}
let mut is_float = false;
let mut is_f32 = false;
self.consume_digits(|c| c.is_ascii_digit());
if self.peek(0) == Some(b'.') && self.peek(1) != Some(b'.') {
is_float = true;
self.pos += 1;
self.consume_digits(|c| c.is_ascii_digit());
}
if matches!(self.peek(0), Some(b'e' | b'E' | b'f'))
&& matches!(self.peek(1), Some(c) if c.is_ascii_digit() || c == b'+' || c == b'-')
{
if self.peek(0) == Some(b'f') {
is_f32 = true;
} else {
is_float = true;
}
self.pos += 1;
if matches!(self.peek(0), Some(b'+' | b'-')) {
self.pos += 1;
}
while matches!(self.peek(0), Some(c) if c.is_ascii_digit()) {
self.pos += 1;
}
}
let kind = if is_f32 {
TokKind::Float32
} else if is_float {
TokKind::Float
} else {
TokKind::Integer
};
self.push(kind, start, self.pos);
}
fn consume_digits(&mut self, is_digit: impl Fn(u8) -> bool) -> bool {
let mut seen = false;
while matches!(self.peek(0), Some(c) if is_digit(c) || c == b'_') {
seen |= self.peek(0) != Some(b'_');
self.pos += 1;
}
seen
}
fn lex_ident_or_keyword(&mut self, start: usize) {
self.scan_ident();
let text = &self.input[start..self.pos];
let keyword = keyword_kind(text);
if keyword.is_none()
&& match self.peek(0) {
Some(b'"') => text != "typegroup",
Some(b'`') => true,
_ => false,
}
{
self.push(TokKind::StringPrefix, start, self.pos);
let open = self.pos;
if self.peek(0) == Some(b'"') {
self.lex_open_string(open, true);
} else {
self.lex_open_cmd(open, true);
}
return;
}
self.push(keyword.unwrap_or(TokKind::Ident), start, self.pos);
}
fn lex_interp_ident(&mut self) {
let start = self.pos;
self.scan_ident();
self.push(TokKind::Ident, start, self.pos);
}
fn scan_ident(&mut self) {
self.pos += self.char_at(self.pos).len_utf8();
loop {
let c = self.char_at(self.pos);
if c == '!' && self.peek(1) == Some(b'=') {
break;
}
if self.pos < self.bytes.len() && is_ident_continue(c) {
self.pos += c.len_utf8();
} else {
break;
}
}
}
}
#[rustfmt::skip]
const OPS: &[(&[u8], TokKind)] = &[
(b".>>>=", TokKind::DotUShrEq),
(b".<-->", TokKind::DotLeftRightArrow),
(b".>>>", TokKind::DotUShr),
(b".//=", TokKind::DotSlashSlashEq),
(b".<<=", TokKind::DotShlEq),
(b".>>=", TokKind::DotShrEq),
(b".-->", TokKind::DotLongArrow),
(b".<--", TokKind::DotLeftLongArrow),
(b".===", TokKind::DotEqEqEq),
(b".!==", TokKind::DotNotEqEq),
(b".+%=", TokKind::DotPlusPercentEq),
(b".-%=", TokKind::DotMinusPercentEq),
(b".*%=", TokKind::DotStarPercentEq),
(b"...", TokKind::DotDotDot),
(b".==", TokKind::DotEqEq),
(b".!=", TokKind::DotNotEq),
(b".<=", TokKind::DotLe),
(b".>=", TokKind::DotGe),
(b".<<", TokKind::DotShl),
(b".>>", TokKind::DotShr),
(b".<:", TokKind::DotSubtype),
(b".>:", TokKind::DotSupertype),
(b".//", TokKind::DotSlashSlash),
(b".**", TokKind::DotStarStar),
(b".--", TokKind::DotMinusMinus),
(b".=>", TokKind::DotFatArrow),
(b".|>", TokKind::DotPipeGt),
(b".&&", TokKind::DotAndAnd),
(b".||", TokKind::DotOrOr),
(b".+%", TokKind::DotPlusPercent),
(b".-%", TokKind::DotMinusPercent),
(b".*%", TokKind::DotStarPercent),
(b".+=", TokKind::DotPlusEq),
(b".-=", TokKind::DotMinusEq),
(b".*=", TokKind::DotStarEq),
(b"./=", TokKind::DotSlashEq),
(b".\\=", TokKind::DotBackslashEq),
(b".^=", TokKind::DotCaretEq),
(b".%=", TokKind::DotPercentEq),
(b".&=", TokKind::DotAmpEq),
(b".|=", TokKind::DotPipeEq),
(b"..", TokKind::DotDot),
(b".+", TokKind::DotPlus),
(b".-", TokKind::DotMinus),
(b".*", TokKind::DotStar),
(b"./", TokKind::DotSlash),
(b".\\", TokKind::DotBackslash),
(b".^", TokKind::DotCaret),
(b".%", TokKind::DotPercent),
(b".=", TokKind::DotEq),
(b".<", TokKind::DotLt),
(b".>", TokKind::DotGt),
(b".~", TokKind::DotTilde),
(b".&", TokKind::DotAmp),
(b".|", TokKind::DotPipe),
(b".!", TokKind::DotBang),
(b".", TokKind::Dot),
(b"<-->", TokKind::LeftRightArrow),
(b"<--", TokKind::LeftLongArrow),
(b"<<=", TokKind::ShlEq),
(b"<=", TokKind::Le),
(b"<:", TokKind::Subtype),
(b"<|", TokKind::PipeLt),
(b"<<", TokKind::Shl),
(b"<", TokKind::Lt),
(b">>>=", TokKind::UShrEq),
(b">>>", TokKind::UShr),
(b">>=", TokKind::ShrEq),
(b">=", TokKind::Ge),
(b">:", TokKind::Supertype),
(b">>", TokKind::Shr),
(b">", TokKind::Gt),
(b"===", TokKind::EqEqEq),
(b"==", TokKind::EqEq),
(b"=>", TokKind::FatArrow),
(b"=", TokKind::Eq),
(b"!==", TokKind::NotEqEq),
(b"!=", TokKind::NotEq),
(b"!", TokKind::Bang),
(b"+%=", TokKind::PlusPercentEq),
(b"+%", TokKind::PlusPercent),
(b"++", TokKind::PlusPlus),
(b"+=", TokKind::PlusEq),
(b"+", TokKind::Plus),
(b"-->", TokKind::LongArrow),
(b"-%=", TokKind::MinusPercentEq),
(b"--", TokKind::MinusMinus),
(b"->", TokKind::Arrow),
(b"-%", TokKind::MinusPercent),
(b"-=", TokKind::MinusEq),
(b"-", TokKind::Minus),
(b"*%=", TokKind::StarPercentEq),
(b"**", TokKind::StarStar),
(b"*%", TokKind::StarPercent),
(b"*=", TokKind::StarEq),
(b"*", TokKind::Star),
(b"//=", TokKind::SlashSlashEq),
(b"//", TokKind::SlashSlash),
(b"/=", TokKind::SlashEq),
(b"/", TokKind::Slash),
(b"\\=", TokKind::BackslashEq),
(b"\\", TokKind::Backslash),
(b"^=", TokKind::CaretEq),
(b"^", TokKind::Caret),
(b"%=", TokKind::PercentEq),
(b"%", TokKind::Percent),
(b"&&", TokKind::AndAnd),
(b"&=", TokKind::AmpEq),
(b"&", TokKind::Amp),
(b"||", TokKind::OrOr),
(b"|>", TokKind::PipeGt),
(b"|=", TokKind::PipeEq),
(b"|", TokKind::Pipe),
(b"::", TokKind::ColonColon),
(b":=", TokKind::ColonEq),
(b":", TokKind::Colon),
(b"$=", TokKind::DollarEq),
(b"$", TokKind::Dollar),
(b"~", TokKind::Tilde),
(b"?", TokKind::Question),
(b"@", TokKind::At),
(b"(", TokKind::LParen),
(b")", TokKind::RParen),
(b"[", TokKind::LBracket),
(b"]", TokKind::RBracket),
(b"{", TokKind::LBrace),
(b"}", TokKind::RBrace),
(b",", TokKind::Comma),
(b";", TokKind::Semicolon),
];
#[derive(Clone, Copy)]
struct OpGroup {
start: u8,
end: u8,
seconds: u128,
}
impl OpGroup {
fn extends(&self, next: Option<&u8>) -> bool {
matches!(next, Some(&b) if b < 128 && self.seconds >> b & 1 == 1)
}
}
const OP_INDEX: [OpGroup; 256] = build_op_index();
const fn build_op_index() -> [OpGroup; 256] {
assert!(OPS.len() < u8::MAX as usize, "OPS outgrew its u8 index");
let mut index = [OpGroup {
start: 0,
end: 0,
seconds: 0,
}; 256];
let mut i = 0;
while i < OPS.len() {
let first = OPS[i].0[0];
assert!(
index[first as usize].start == index[first as usize].end,
"OPS entries sharing a first byte must be contiguous"
);
let start = i;
let mut len = usize::MAX;
let mut seconds = 0u128;
while i < OPS.len() && OPS[i].0[0] == first {
let text = OPS[i].0;
assert!(
text.len() <= len,
"OPS entries must run longest-first within a first-byte group"
);
len = text.len();
if len > 1 {
assert!(
text[1] < 128,
"OPS holds ASCII spellings only; a Unicode operator belongs in `unicode_op_at`"
);
seconds |= 1u128 << text[1];
}
i += 1;
}
assert!(
len == 1,
"every OPS group must end with its own single-byte spelling"
);
index[first as usize] = OpGroup {
start: start as u8,
end: i as u8,
seconds,
};
}
index
}
fn try_ascii_op(rest: &[u8]) -> Option<(TokKind, usize)> {
let group = &OP_INDEX[*rest.first()? as usize];
let entries = &OPS[group.start as usize..group.end as usize];
if !group.extends(rest.get(1)) {
let (text, kind) = *entries.last()?;
return Some((kind, text.len()));
}
let next = rest[1]; entries
.iter()
.find(|(text, _)| match text.get(1) {
Some(&second) => second == next && rest.starts_with(text),
None => true,
})
.map(|&(text, kind)| (kind, text.len()))
}
macro_rules! define_keyword_tables {
($($text:literal $tok:ident $syn:ident,)*) => {
pub const KEYWORDS: &[&str] = &[$($text),*];
fn keyword_kind(text: &str) -> Option<TokKind> {
Some(match text {
$($text => TokKind::$tok,)*
_ => return None,
})
}
impl TokKind {
pub(crate) fn is_keyword(self) -> bool {
matches!(self, $(TokKind::$tok)|*)
}
}
};
}
keyword_table!(define_keyword_tables);
pub fn is_ident_start(c: char) -> bool {
if c.is_ascii() {
c == '_' || c.is_ascii_alphabetic()
} else {
c != '\u{1f8b2}' && super::unicode_ident::is_unicode_ident_start(c)
}
}
pub fn is_ident_continue(c: char) -> bool {
if c.is_ascii() {
c == '_' || c == '!' || c.is_ascii_alphanumeric()
} else {
super::unicode_ident::is_unicode_ident_continue(c)
}
}
const OP_SUFFIX_CHARS: &str = "²³¹ʰʲʳʷʸˡˢˣ˱˲ᴬᴮᴰᴱᴳᴴᴵᴶᴷᴸᴹᴺᴼᴾᴿᵀᵁᵂᵃᵅᵇᵈᵉᵋᵍᵏᵐᵒᵖᵗᵘᵛᵝᵞᵟᵠᵡᵢᵣᵤᵥᵦᵧᵨᵩᵪᶜᶠᶥᶦᶫᶰᶲᶸᶻᶿ′″‴‵‶‷⁗⁰ⁱ⁴⁵⁶⁷⁸⁹⁺⁻⁼⁽⁾ⁿ₀₁₂₃₄₅₆₇₈₉₊₋₌₍₎ₐₑₒₓₔₕₖₗₘₙₚₛₜⱼⱽꜛꜜꜝ";
pub(crate) fn is_op_suffix_char(c: char) -> bool {
OP_SUFFIX_CHARS.contains(c)
}
fn op_takes_suffix(kind: TokKind) -> bool {
use TokKind::*;
matches!(
kind,
Plus | Minus
| Star
| Slash
| Backslash
| SlashSlash
| Caret
| Percent
| PlusPercent
| MinusPercent
| StarPercent
| EqEq
| NotEq
| EqEqEq
| NotEqEq
| Lt
| Le
| Gt
| Ge
| Amp
| Pipe
| Shl
| Shr
| UShr
| PipeGt
| PipeLt
| FatArrow
| LongArrow
| LeftRightArrow
| LeftLongArrow
| Transpose
| DotPlus
| DotMinus
| DotStar
| DotPlusPercent
| DotMinusPercent
| DotStarPercent
| DotSlash
| DotBackslash
| DotSlashSlash
| DotCaret
| DotPercent
| DotEqEq
| DotNotEq
| DotEqEqEq
| DotNotEqEq
| DotLt
| DotLe
| DotGt
| DotGe
| DotShl
| DotShr
| DotUShr
| DotFatArrow
| DotLongArrow
| DotLeftLongArrow
| DotLeftRightArrow
| DotPipeGt
| DotAmp
| DotPipe
| UniArrow
| UniComparison
| UniColon
| UniPlus
| UniTimes
| UniPower
)
}
const MINUS_SIGN: char = '\u{2212}';
const DIVIDE_SIGN: char = '\u{f7}';
const XOR_SIGN: char = '\u{22bb}';
fn is_unicode_infix_tier(kind: TokKind) -> bool {
use TokKind::*;
matches!(
kind,
UniArrow | UniComparison | UniColon | UniPlus | UniTimes | UniPower
)
}
#[cfg(test)]
mod tests {
use super::*;
fn kinds(input: &str) -> Vec<TokKind> {
lex(input).into_iter().map(|t| t.kind).collect()
}
fn roundtrips(input: &str) -> bool {
let joined: String = lex(input).into_iter().map(|t| t.text).collect();
joined == input
}
#[test]
fn every_keyword_lexes_as_one_keyword_token() {
for kw in KEYWORDS {
assert_eq!(kinds(kw), vec![keyword_kind(kw).unwrap()], "lexing {kw}");
assert!(keyword_kind(kw).unwrap().is_keyword(), "{kw} is a keyword");
assert!(
crate::parser::tree_builder::syntax_kind_for(keyword_kind(kw).unwrap())
.is_keyword(),
"{kw} materializes as a keyword kind"
);
}
}
#[test]
fn the_wide_arrow_operator_is_not_an_identifier_start() {
assert_eq!(
kinds("x \u{1f8b2} y"),
vec![
TokKind::Ident,
TokKind::Whitespace,
TokKind::UniArrow,
TokKind::Whitespace,
TokKind::Ident,
]
);
assert_eq!(kinds("a\u{1f8b2}b"), vec![TokKind::Ident]);
}
#[test]
fn lossless_over_assorted_input() {
for input in [
"x = 1 + 2\n",
"f(a, b)\n",
"#= a #= nested =# b =#\n",
"function g(x)\n x ^ 2\nend\n",
"s = \"hello\\n\"\nc = 'a'\n",
"if a >= b\n a\nelseif c\n c\nelse\n b\nend\n",
"for i in 1:10\n x\nend\n",
"while x > 0\n x\nend\n",
"map(xs) do x, y\n x + y\nend\n",
"try\n f()\ncatch e\n g()\nfinally\n h()\nend\n",
"mutable struct Counter\n n\nend\n",
"module M\nend\n",
"α = β + 1\n",
"0x1f + 0b1010\n",
"x = 0o755\ny = 0x1.8p3\nz = 1f0\n",
"r = 3//4 + 1_000\nq = a .// b\n",
"n = 1.5e-3\nm = .5\nk = 2.\n",
"s = \"a$x b\"\n",
"s = \"a$(f(x))b\"\n",
"s = \"\"\"x$(y)\"\"\"\n",
"c = `echo $x`\n",
"r = raw\"\\d+\"\n",
"m = r\"pat\"ims\n",
"v = v\"1.2.3\"\n",
"b = b\"\\x00\"\n",
"s = \"$foo.bar\"\n",
"s = \"\\$lit\"\n",
"s = \"$$\"\n",
"s = \"unterminated\n",
"s = \"$(g(\"nested\"))\"\n",
"s = \"a\\αb\"\n",
"s = \"\\α",
] {
assert!(roundtrips(input), "did not round-trip: {input:?}");
}
}
#[test]
fn interpolation_with_bare_ident() {
assert_eq!(
kinds("\"a$x\""),
vec![
TokKind::StringDelimOpen,
TokKind::StringContent,
TokKind::Dollar,
TokKind::Ident,
TokKind::StringDelimClose,
]
);
}
#[test]
fn interpolation_with_parenthesized_expr() {
assert_eq!(
kinds("\"$(y)\""),
vec![
TokKind::StringDelimOpen,
TokKind::Dollar,
TokKind::LParen,
TokKind::Ident,
TokKind::RParen,
TokKind::StringDelimClose,
]
);
}
#[test]
fn nested_parens_in_interpolation() {
assert_eq!(
kinds("\"$(f(x))\""),
vec![
TokKind::StringDelimOpen,
TokKind::Dollar,
TokKind::LParen,
TokKind::Ident,
TokKind::LParen,
TokKind::Ident,
TokKind::RParen,
TokKind::RParen,
TokKind::StringDelimClose,
]
);
}
#[test]
fn raw_literal_does_not_interpolate() {
assert_eq!(
kinds("raw\"$x\\d\""),
vec![
TokKind::StringPrefix,
TokKind::StringDelimOpen,
TokKind::StringContent,
TokKind::StringDelimClose,
]
);
}
#[test]
fn prefix_and_suffix_flags() {
assert_eq!(
kinds("r\"pat\"ims"),
vec![
TokKind::StringPrefix,
TokKind::StringDelimOpen,
TokKind::StringContent,
TokKind::StringDelimClose,
TokKind::StringSuffix,
]
);
}
#[test]
fn command_literal_interpolates() {
assert_eq!(
kinds("`cmd $x`"),
vec![
TokKind::CmdDelimOpen,
TokKind::StringContent,
TokKind::Dollar,
TokKind::Ident,
TokKind::CmdDelimClose,
]
);
}
#[test]
fn escaped_dollar_is_content() {
assert_eq!(
kinds("\"\\$x\""),
vec![
TokKind::StringDelimOpen,
TokKind::StringContent,
TokKind::StringDelimClose,
]
);
}
#[test]
fn plain_string_adjacent_ident_is_not_a_suffix() {
assert_eq!(
kinds("\"a\"b"),
vec![
TokKind::StringDelimOpen,
TokKind::StringContent,
TokKind::StringDelimClose,
TokKind::Ident,
]
);
}
#[test]
fn prefix_requires_adjacent_quote() {
assert_eq!(
kinds("r \"x\""),
vec![
TokKind::Ident,
TokKind::Whitespace,
TokKind::StringDelimOpen,
TokKind::StringContent,
TokKind::StringDelimClose,
]
);
}
#[test]
fn keywords_and_operators() {
assert_eq!(
kinds("a == b"),
vec![
TokKind::Ident,
TokKind::Whitespace,
TokKind::EqEq,
TokKind::Whitespace,
TokKind::Ident
]
);
assert_eq!(keyword_kind("function"), Some(TokKind::FunctionKw));
assert_eq!(keyword_kind("ends"), None);
}
#[test]
fn bang_in_identifier() {
assert_eq!(kinds("push!"), vec![TokKind::Ident]);
assert_eq!(kinds("a!b"), vec![TokKind::Ident]);
assert_eq!(
kinds("a!=b"),
vec![TokKind::Ident, TokKind::NotEq, TokKind::Ident]
);
assert_eq!(
kinds("a!==b"),
vec![TokKind::Ident, TokKind::NotEqEq, TokKind::Ident]
);
assert_eq!(
kinds("a!!=b"),
vec![TokKind::Ident, TokKind::NotEq, TokKind::Ident]
);
}
#[test]
fn unicode_identifier_chars() {
assert_eq!(kinds("x\u{304}"), vec![TokKind::Ident]);
assert_eq!(kinds("x\u{302}r"), vec![TokKind::Ident]);
assert_eq!(kinds("\u{3b1}\u{2032}"), vec![TokKind::Ident]);
assert_eq!(kinds("\u{3c1}\u{221e}"), vec![TokKind::Ident]);
assert_eq!(kinds("\u{2207}batchnorm"), vec![TokKind::Ident]);
assert_eq!(kinds("\u{2081}"), vec![TokKind::ErrorIdentifierStart]);
assert!(roundtrips("x\u{304} = 1"));
assert!(roundtrips("\u{2207}f(x) = x"));
}
#[test]
fn unicode_17_identifier_chars() {
assert_eq!(kinds("\u{88f}"), vec![TokKind::Ident]);
assert_eq!(kinds("\u{11db0}"), vec![TokKind::Ident]);
assert_eq!(kinds("x\u{11b60}"), vec![TokKind::Ident]);
assert_eq!(kinds("\u{11b60}"), vec![TokKind::ErrorIdentifierStart]);
assert!(roundtrips("\u{88f} = \u{11db0} + x\u{11b60}"));
}
#[test]
fn unicode_18_identifier_chars() {
for c in [
'\u{558}',
'\u{209d}',
'\u{209e}',
'\u{209f}',
'\u{a7dd}',
'\u{107bb}',
'\u{10ed9}',
'\u{12550}',
'\u{18e00}',
'\u{1dfd0}',
'\u{3d000}',
'\u{3fc3f}',
] {
for input in [c.to_string(), format!("x{c}!")] {
assert_eq!(kinds(&input), vec![TokKind::Ident], "{input:?}");
assert!(roundtrips(&input));
}
}
for c in [
'\u{5c8}',
'\u{b53}',
'\u{1ade}',
'\u{10ec9}',
'\u{11df0}',
'\u{1d127}',
'\u{1d250}',
] {
assert_eq!(kinds(&format!("x{c}")), vec![TokKind::Ident], "{c:?}");
assert_eq!(
kinds(&c.to_string()),
vec![TokKind::ErrorIdentifierStart],
"{c:?}"
);
assert!(roundtrips(&format!("{c} x{c}")));
}
for c in ['\u{3cfff}', '\u{3fc40}'] {
assert_eq!(kinds(&c.to_string()), vec![TokKind::Unknown], "{c:?}");
}
}
#[test]
fn unicode_18_operator_suffixes() {
for suffix in "ᵅᵋᶲ˱˲ₔ\u{1dfd0}".chars() {
for (op, kind) in [
("+", TokKind::Plus),
(".+", TokKind::DotPlus),
("⊗", TokKind::UniTimes),
(".⊗", TokKind::UniTimes),
("-->", TokKind::LongArrow),
("'", TokKind::Transpose),
] {
let operator = format!("{op}{suffix}");
let input = format!("a{operator}b");
let tokens = lex(&input);
assert_eq!(tokens.len(), 3, "{input:?}: {tokens:?}");
assert_eq!(tokens[1].kind, kind, "{input:?}");
assert_eq!(tokens[1].text, operator, "{input:?}");
assert_eq!(tokens[1].start, 1);
assert_eq!(tokens[1].end, 1 + operator.len());
assert_eq!(tokens[2].text, "b");
assert!(roundtrips(&input));
}
for op in ["=", ":", "::", "&&", "||", "->", "<:", "!", "~", "+="] {
let input = format!("{op}{suffix}");
assert_eq!(lex(&input)[0].text, op, "{input:?}");
assert!(roundtrips(&input));
}
}
}
#[test]
fn identity_operators() {
assert_eq!(
kinds("a===b"),
vec![TokKind::Ident, TokKind::EqEqEq, TokKind::Ident]
);
assert_eq!(
kinds("a==b"),
vec![TokKind::Ident, TokKind::EqEq, TokKind::Ident]
);
}
#[test]
fn numeric_literal_kinds() {
assert_eq!(kinds("123"), vec![TokKind::Integer]);
assert_eq!(kinds("1_000"), vec![TokKind::Integer]);
assert_eq!(
kinds("12345678901234567890123456789"),
vec![TokKind::Integer]
);
assert_eq!(kinds("0x1f"), vec![TokKind::HexInt]);
assert_eq!(kinds("0o755"), vec![TokKind::OctInt]);
assert_eq!(kinds("0b1010"), vec![TokKind::BinInt]);
assert_eq!(kinds("3.14"), vec![TokKind::Float]);
assert_eq!(kinds(".5"), vec![TokKind::Float]);
assert_eq!(kinds("2."), vec![TokKind::Float]);
assert_eq!(kinds("1.5e-3"), vec![TokKind::Float]);
assert_eq!(kinds("1f0"), vec![TokKind::Float32]);
assert_eq!(kinds("2.5f-3"), vec![TokKind::Float32]);
assert_eq!(kinds("0x1p0"), vec![TokKind::Float]);
assert_eq!(kinds("0x1.8p3"), vec![TokKind::Float]);
assert_eq!(kinds("0x1.p3"), vec![TokKind::Float]);
assert_eq!(kinds("0x.8p3"), vec![TokKind::Float]);
assert_eq!(kinds("0x1P+2"), vec![TokKind::Float]);
assert_eq!(kinds("0x1_2p3"), vec![TokKind::Float]);
assert_eq!(kinds("0x1e3"), vec![TokKind::HexInt]);
}
#[test]
fn malformed_hex_literal_kinds() {
for src in [
"0x1p", "0x1p+", "0x1.8p", "0x1.p", "0x.p3", "0xp3", "0x", "0x_",
] {
assert_eq!(
kinds(src),
vec![TokKind::ErrorInvalidNumber],
"{src} should be an invalid numeric constant"
);
}
for src in ["0x1.8", "0x1.", "0x.8", "0x.", "0x1.8e2"] {
assert_eq!(
kinds(src),
vec![TokKind::ErrorHexFloatNoP],
"{src} should require a p exponent"
);
}
}
#[test]
fn rational_operators() {
assert_eq!(
kinds("3//4"),
vec![TokKind::Integer, TokKind::SlashSlash, TokKind::Integer]
);
assert_eq!(
kinds("a .// b"),
vec![
TokKind::Ident,
TokKind::Whitespace,
TokKind::DotSlashSlash,
TokKind::Whitespace,
TokKind::Ident,
]
);
}
#[test]
fn left_division_operators() {
assert_eq!(
kinds("a\\b"),
vec![TokKind::Ident, TokKind::Backslash, TokKind::Ident]
);
assert_eq!(
kinds("a\\=b"),
vec![TokKind::Ident, TokKind::BackslashEq, TokKind::Ident]
);
assert_eq!(
kinds("a .\\ b"),
vec![
TokKind::Ident,
TokKind::Whitespace,
TokKind::DotBackslash,
TokKind::Whitespace,
TokKind::Ident,
]
);
assert_eq!(
kinds("a.\\=b"),
vec![TokKind::Ident, TokKind::DotBackslashEq, TokKind::Ident]
);
}
#[test]
fn left_arrow_operators() {
assert_eq!(
kinds("a<--b"),
vec![TokKind::Ident, TokKind::LeftLongArrow, TokKind::Ident]
);
assert_eq!(
kinds("a<-->b"),
vec![TokKind::Ident, TokKind::LeftRightArrow, TokKind::Ident]
);
assert_eq!(
kinds("a.<--b"),
vec![TokKind::Ident, TokKind::DotLeftLongArrow, TokKind::Ident]
);
assert_eq!(
kinds("a.<-->b"),
vec![TokKind::Ident, TokKind::DotLeftRightArrow, TokKind::Ident]
);
assert_eq!(
kinds("a<-b"),
vec![TokKind::Ident, TokKind::Lt, TokKind::Minus, TokKind::Ident]
);
}
#[test]
fn compound_shift_and_unicode_assignments() {
assert_eq!(
kinds("a<<=b"),
vec![TokKind::Ident, TokKind::ShlEq, TokKind::Ident]
);
assert_eq!(
kinds("a>>=b"),
vec![TokKind::Ident, TokKind::ShrEq, TokKind::Ident]
);
assert_eq!(
kinds("a>>>=b"),
vec![TokKind::Ident, TokKind::UShrEq, TokKind::Ident]
);
assert_eq!(
kinds("a÷=b"),
vec![TokKind::Ident, TokKind::DivEq, TokKind::Ident]
);
assert_eq!(
kinds("a⊻=b"),
vec![TokKind::Ident, TokKind::XorEq, TokKind::Ident]
);
assert_eq!(
kinds("a.<<=b"),
vec![TokKind::Ident, TokKind::DotShlEq, TokKind::Ident]
);
assert_eq!(
kinds("a.>>>=b"),
vec![TokKind::Ident, TokKind::DotUShrEq, TokKind::Ident]
);
assert_eq!(
kinds("a.÷=b"),
vec![TokKind::Ident, TokKind::DotDivEq, TokKind::Ident]
);
assert_eq!(
kinds("a.⊻=b"),
vec![TokKind::Ident, TokKind::DotXorEq, TokKind::Ident]
);
}
#[test]
fn uppercase_hex_prefix_is_not_a_literal() {
assert_eq!(kinds("0X1"), vec![TokKind::Integer, TokKind::Ident]);
}
#[test]
fn inf_and_nan_are_identifiers() {
assert_eq!(kinds("Inf"), vec![TokKind::Ident]);
assert_eq!(kinds("NaN"), vec![TokKind::Ident]);
assert_eq!(kinds("Inf32"), vec![TokKind::Ident]);
}
#[test]
fn subtype_and_supertype_operators() {
assert_eq!(
kinds("T<:U"),
vec![TokKind::Ident, TokKind::Subtype, TokKind::Ident]
);
assert_eq!(
kinds("T>:U"),
vec![TokKind::Ident, TokKind::Supertype, TokKind::Ident]
);
}
#[test]
fn splat_is_three_dots() {
assert_eq!(kinds("x..."), vec![TokKind::Ident, TokKind::DotDotDot]);
assert_eq!(kinds(".."), vec![TokKind::DotDot]);
assert_eq!(
kinds("a.b"),
vec![TokKind::Ident, TokKind::Dot, TokKind::Ident]
);
}
#[test]
fn broadcasting_operators() {
assert_eq!(
kinds("a .+ b"),
vec![
TokKind::Ident,
TokKind::Whitespace,
TokKind::DotPlus,
TokKind::Whitespace,
TokKind::Ident
]
);
assert_eq!(kinds("x .== y").get(2), Some(&TokKind::DotEqEq));
assert_eq!(kinds("x .= y").get(2), Some(&TokKind::DotEq));
assert_eq!(kinds("a .<= b").get(2), Some(&TokKind::DotLe));
assert_eq!(kinds("x .=== y").get(2), Some(&TokKind::DotEqEqEq));
assert_eq!(kinds("x .!== y").get(2), Some(&TokKind::DotNotEqEq));
assert_eq!(kinds(".!y").first(), Some(&TokKind::DotBang));
assert_eq!(kinds("x .!= y").get(2), Some(&TokKind::DotNotEq));
assert_eq!(kinds(".√[3]").first(), Some(&TokKind::UniRadical));
assert_eq!(kinds(".¬a").first(), Some(&TokKind::UniRadical));
assert_eq!(kinds("a .&= b").get(2), Some(&TokKind::DotAmpEq));
assert_eq!(kinds("a .|= b").get(2), Some(&TokKind::DotPipeEq));
assert_eq!(kinds("a .& b").get(2), Some(&TokKind::DotAmp));
assert_eq!(kinds("a .| b").get(2), Some(&TokKind::DotPipe));
assert_eq!(kinds("a .|> b").get(2), Some(&TokKind::DotPipeGt));
assert_eq!(
kinds("a.b"),
vec![TokKind::Ident, TokKind::Dot, TokKind::Ident]
);
assert_eq!(kinds(".."), vec![TokKind::DotDot]);
assert_eq!(
kinds("f.(x)"),
vec![
TokKind::Ident,
TokKind::Dot,
TokKind::LParen,
TokKind::Ident,
TokKind::RParen
]
);
}
#[test]
fn where_is_a_keyword() {
assert_eq!(keyword_kind("where"), Some(TokKind::WhereKw));
assert_eq!(kinds("where"), vec![TokKind::WhereKw]);
}
#[test]
fn every_ops_entry_lexes_as_itself() {
for &(text, kind) in OPS {
let spelling = std::str::from_utf8(text).expect("OPS spellings are ASCII");
let tokens = lex(spelling);
assert_eq!(
tokens.iter().map(|t| t.kind).collect::<Vec<_>>(),
vec![kind],
"{spelling:?} did not lex as one {kind:?}"
);
assert_eq!(tokens[0].text, spelling);
}
}
#[test]
fn longest_match_beats_every_shared_prefix() {
for spelling in [
".>>>=", ".<-->", ".<--", ".//=", ".===", ".!==", "...", "<-->", ">>>=", "-->", "//=",
"+%=", ".+%", ".-%", ".*%", ".+%=", ".-%=", ".*%=", "!==", "===",
] {
assert_eq!(kinds(spelling).len(), 1, "{spelling:?} was split");
}
assert_eq!(kinds("<-"), vec![TokKind::Lt, TokKind::Minus]);
assert_eq!(kinds("+++"), vec![TokKind::PlusPlus, TokKind::Plus]);
assert_eq!(kinds("++="), vec![TokKind::PlusPlus, TokKind::Eq]);
assert_eq!(
kinds("a.b"),
vec![TokKind::Ident, TokKind::Dot, TokKind::Ident]
);
}
#[test]
fn unicode_operators_beat_the_ascii_table() {
assert_eq!(kinds(".×"), vec![TokKind::UniTimes]);
assert_eq!(kinds(".√"), vec![TokKind::UniRadical]);
assert_eq!(kinds(".÷="), vec![TokKind::DotDivEq]);
assert_eq!(kinds(".⊻="), vec![TokKind::DotXorEq]);
assert_eq!(kinds("−"), vec![TokKind::Minus]);
assert_eq!(kinds("−="), vec![TokKind::MinusEq]);
assert_eq!(kinds(".−="), vec![TokKind::DotMinusEq]);
assert_eq!(kinds("÷="), vec![TokKind::DivEq]);
assert_eq!(kinds(".⩴"), vec![TokKind::Dot, TokKind::UniAssign]);
}
}