use std::collections::HashMap;
use logos::Logos;
use crate::parsing::lexer::LexingErrorKind;
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub struct DataId(pub usize);
#[derive(Debug, Clone)]
pub struct TokensData {
pub ints128: Vec<i128>,
pub strings: Vec<String>,
}
pub const STRING_ID_OFFSET: usize = usize::MAX / 2;
impl TokensData {
pub fn get_i128(&self, id: DataId) -> i128 {
self.ints128
.get(id.0)
.expect("invalid int128 DataId")
.to_owned()
}
pub fn get_string(&self, id: DataId) -> &str {
id.0.checked_sub(STRING_ID_OFFSET)
.and_then(|idx| self.strings.get(idx))
.expect("invalid string DataId")
}
}
#[derive(Debug)]
pub struct TokensExtras {
pub ints128: HashMap<i128, DataId>,
pub strings: HashMap<String, DataId>,
_next_ints128_data_id: DataId,
_next_string_data_id: DataId,
}
impl TokensExtras {
fn string_id(&mut self, string: &str) -> DataId {
self.strings.get(string).copied().unwrap_or_else(|| {
let id = self._next_string_data_id;
self._next_string_data_id.0 += 1;
self.strings.insert(string.to_string(), id);
id
})
}
fn int128_id(&mut self, value: i128) -> DataId {
self.ints128.get(&value).copied().unwrap_or_else(|| {
let id = self._next_ints128_data_id;
self._next_ints128_data_id.0 += 1;
self.ints128.insert(value, id);
id
})
}
}
impl Default for TokensExtras {
fn default() -> Self {
Self {
ints128: HashMap::new(),
strings: HashMap::new(),
_next_ints128_data_id: DataId(0),
_next_string_data_id: DataId(usize::MAX / 2),
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Logos)]
#[logos(error = LexingErrorKind)]
#[logos(skip r"[ \t\r\n\f]+")]
#[logos(skip(r"//[^\n\r]*", allow_greedy = true))]
#[logos(skip r"/\*([^*]|\*[^/])*\*/")]
#[logos(extras = TokensExtras)]
pub enum Token {
#[token("unsafe")]
UnsafeKw,
#[token("pub")]
PubKw,
#[token("fn")]
FnKw,
#[token("type")]
TypeKw,
#[token("struct")]
StructKw,
#[token("enum")]
EnumKw,
#[token("trait")]
TraitKw,
#[token("dyn")]
DynKw,
#[token("impl")]
ImplKw,
#[token("extension")]
ExtensionKw,
#[token("patch")]
PatchKw,
#[token("as")]
AsKw,
#[token("from")]
FromKw,
#[token("const")]
ConstKw,
#[token("mut")]
MutKw,
#[token("acs")]
AcsKw,
#[token("fld")]
FldKw,
#[token("var")]
VarKw,
#[token("let")]
LetKw,
#[token("if")]
IfKw,
#[token("match")]
MatchKw,
#[token("else")]
ElseKw,
#[token("loop")]
LoopKw,
#[token("while")]
WhileKw,
#[token("for")]
ForKw,
#[token("in")]
InKw,
#[token("return")]
ReturnKw,
#[token("self")]
LowSelfKw,
#[token("Self")]
BigSelfKw,
#[token("true")]
TrueKw,
#[token("false")]
FalseKw,
#[token(".")]
Dot,
#[token(",")]
Comma,
#[token(":")]
Colon,
#[token(";")]
Semicolon,
#[token("'")]
SingleQuote,
#[token("\"")]
DoubleQuote,
#[token("->")]
Arrow,
#[token("=>")]
FatArrow,
#[token("&")]
Ampersand,
#[token("=")]
Eq,
#[token("==")]
EqEq,
#[token("!=")]
NotEq,
#[token("!")]
Bang,
#[token("+")]
Plus,
#[token("+=")]
PlusEq,
#[token("-")]
Minus,
#[token("-=")]
MinusEq,
#[token("*")]
Asterisk,
#[token("*=")]
AsteriskEq,
#[token("/")]
Slash,
#[token("/=")]
SlashEq,
#[token("<<=")]
ShiftLeftEq,
#[token(">>=")]
ShiftRightEq,
#[token("<")]
LeftAngle,
#[token("<=")]
LeftAngleEq,
#[token(">")]
RightAngle,
#[token(">=")]
RightAngleEq,
#[token("(")]
OpenParen,
#[token(")")]
CloseParen,
#[token("[")]
OpenBracket,
#[token("]")]
CloseBracket,
#[token("{")]
OpenBrace,
#[token("}")]
CloseBrace,
#[regex(r"[0-9][0-9_]*u8", |lex| parse_number::<u8>(lex.slice(), "u8", LexingErrorKind::InvalidUint8Value))]
Uint8(u8),
#[regex(r"[0-9][0-9_]*u16", |lex| parse_number::<u16>(lex.slice(), "u16", LexingErrorKind::InvalidUint16Value))]
Uint16(u16),
#[regex(r"[0-9][0-9_]*u32", |lex| parse_number::<u32>(lex.slice(), "u32", LexingErrorKind::InvalidUint32Value))]
Uint32(u32),
#[regex(r"[0-9][0-9_]*u64", |lex| parse_number::<u64>(lex.slice(), "u64", LexingErrorKind::InvalidUint64Value))]
Uint64(u64),
#[regex(r"[0-9][0-9_]*usize", |lex| parse_number::<usize>(lex.slice(), "usize", LexingErrorKind::InvalidUsizeValue))]
Usize(usize),
#[regex(r"[0-9][0-9_]*i8", |lex| parse_number::<i8>(lex.slice(), "i8", LexingErrorKind::InvalidInt8Value))]
Int8(i8),
#[regex(r"[0-9][0-9_]*i16", |lex| parse_number::<i16>(lex.slice(), "i16", LexingErrorKind::InvalidInt16Value))]
Int16(i16),
#[regex(r"[0-9][0-9_]*i32", |lex| parse_number::<i32>(lex.slice(), "i32", LexingErrorKind::InvalidInt32Value))]
Int32(i32),
#[regex(r"[0-9][0-9_]*i64", |lex| parse_number::<i64>(lex.slice(), "i64", LexingErrorKind::InvalidInt64Value))]
Int64(i64),
#[regex(r"[0-9][0-9_]*isize", |lex| parse_number::<isize>(lex.slice(), "isize", LexingErrorKind::InvalidIsizeValue))]
Isize(isize),
#[regex(r"[0-9][0-9_]*\.[0-9][0-9_]*f32", |lex| parse_number::<f32>(lex.slice(), "f32", LexingErrorKind::InvalidFloat32Value))]
Float32(f32),
#[regex(r"[0-9][0-9_]*\.[0-9][0-9_]*f64", |lex| parse_number::<f64>(lex.slice(), "f64", LexingErrorKind::InvalidFloat64Value))]
Float64(f64),
#[regex(r"[0-9][0-9_]*\.[0-9][0-9_]*", |lex| parse_number::<f64>(lex.slice(), "", LexingErrorKind::InvalidFloatAnyValue))]
FloatAny(f64),
#[regex(r"[0-9][0-9_]*", |lex| parse_number::<i128>(lex.slice(), "", LexingErrorKind::InvalidIntAnyValue).map(|value| lex.extras.int128_id(value)))]
IntAny(DataId),
#[regex(r#"'([^'\\]|\\.)'"#, |lex| handle_char(lex))]
Char(char),
#[regex(r#""([^"\\]|\\.)*""#, |lex| {
let string = &handle_string(lex);
lex.extras.string_id(string)
})]
#[regex(r#"r"([^"]*)""#, |lex| {
let slice = lex.slice();
lex.extras.string_id(&slice[2..slice.len() - 1])
})]
#[regex(r##"r#"([^"]*)"#"##, |lex| {
let slice = lex.slice();
lex.extras.string_id(&slice[3..slice.len() - 2])
})]
String(DataId),
#[regex(r"[A-Za-z_][A-Za-z0-9_]*", |lex| {
lex.extras.string_id(lex.slice())
})]
Ident(DataId),
#[regex(r"'[A-Za-z_][A-Za-z0-9_]*", |lex| {
lex.extras.string_id(&lex.slice()[1..])
})]
Lifetime(DataId),
Eof,
}
fn parse_number<T: std::str::FromStr>(
slice: &str,
suffix: &str,
error: LexingErrorKind,
) -> Result<T, LexingErrorKind> {
let digits = slice
.strip_suffix(suffix)
.expect("numeric lexer rule must match its suffix")
.replace('_', "");
digits.parse().map_err(|_| error)
}
fn handle_char(lex: &mut logos::Lexer<Token>) -> char {
let slice = lex.slice();
let inner = &slice[1..slice.len() - 1];
if let Some(escaped) = inner.strip_prefix('\\') {
match escaped {
"n" => '\n',
"t" => '\t',
"r" => '\r',
"\\" => '\\',
"'" => '\'',
"\"" => '"',
other => other.chars().next().unwrap_or_default(),
}
} else {
inner.chars().next().unwrap_or_default()
}
}
fn handle_string(lex: &mut logos::Lexer<Token>) -> String {
let slice = lex.slice();
let inner = &slice[1..slice.len() - 1];
let mut result = String::with_capacity(inner.len());
let mut chars = inner.chars().peekable();
while let Some(c) = chars.next() {
if c == '\\' {
match chars.next() {
Some('n') => result.push('\n'),
Some('t') => result.push('\t'),
Some('r') => result.push('\r'),
Some('\\') => result.push('\\'),
Some('"') => result.push('"'),
Some(other) => result.push(other),
None => {}
}
} else {
result.push(c);
}
}
result
}