use std::range::Range;
use crate::{
diagnostic,
diagnostics::ErrorKind,
lexer::token::{Token, TokenStream, TokenType},
source::SourceContext,
};
pub(crate) mod tests;
pub(crate) mod token;
pub(crate) const KEYSMASH_MAX_LEN: u8 = 128;
pub(crate) struct Lexer<'lx> {
ctx: &'lx mut SourceContext,
chars: Vec<char>,
start: usize,
current: usize,
start_byte: usize,
current_byte: usize,
tokens: TokenStream,
has_comments: bool,
}
impl<'lx> Lexer<'lx> {
pub(crate) fn new(ctx: &'lx mut SourceContext) -> Self {
let chars = ctx.source.chars().collect();
Self {
ctx,
chars,
start: 0,
current: 0,
start_byte: 0,
current_byte: 0,
tokens: TokenStream::default(),
has_comments: false,
}
}
pub(crate) fn tokens(self) -> TokenStream {
self.tokens
}
fn tokenise(&mut self, kind: TokenType, lexeme: String) {
let range = self.byte_range();
self.tokens.push(Token::new(kind, &lexeme, range));
self.start = self.current;
self.start_byte = self.current_byte;
}
fn is_valid_keysmash_char(&self, c: char) -> bool {
c.is_ascii_alphabetic() || (192..=255).contains(&(c as u32))
}
fn lexeme(&self) -> String {
self.chars[self.range()].iter().collect()
}
fn range(&'lx self) -> Range<usize> {
(self.start..self.current).into()
}
fn byte_range(&'lx self) -> Range<usize> {
(self.start_byte..self.current_byte).into()
}
fn peek(&mut self) -> Option<char> {
self.chars.get(self.current).copied()
}
fn next(&mut self) -> Option<char> {
let peek = self.peek()?;
self.current += 1;
self.current_byte += peek.len_utf8();
Some(peek)
}
fn check(&mut self, expected: char) -> bool {
self.peek().is_some_and(|peek| peek == expected)
}
fn matches(&mut self, expected: char) -> bool {
if self.check(expected) {
self.next();
return true;
}
false
}
pub(crate) fn lex_tokens(&mut self, in_repl: bool) {
while let Some(char) = self.next() {
let token_type = match char {
'>' => self.lex_flustered_emoticon(),
'@' | 'O' => self.lex_heavy_flustered_emoticon(char),
'^' => self.lex_happy_emoticon(),
':' => self.lex_colon_three(true),
'🥺' => TokenType::Sub,
'👉' => {
if self.matches('👈') {
TokenType::Point
} else {
self.ctx.report(diagnostic!(
ErrorKind::UnfinishedEmoticon {
praise_term: self.ctx.rand_praise_term().into(),
petname: self.ctx.rand_petname().into(),
char_to_add: '👈'
},
labels = [(self.byte_range(), "")]
));
TokenType::Error
}
}
'🏳' => {
self.lex_flag_emoji();
continue;
}
c if c.is_whitespace() || c == ';' => {
self.start = self.current;
self.start_byte = self.current_byte;
continue;
}
c => self.lex_keysmash(c),
};
self.tokenise(token_type, self.lexeme());
}
if !self.has_comments && !in_repl {
self.ctx.report(diagnostic!(
ErrorKind::UncommentedSource {
interp_title: self.ctx.rand_interp_title().into(),
petname: self.ctx.rand_petname().into(),
praise_term: self.ctx.rand_praise_term().into(),
},
labels = [(self.byte_range(), "")]
))
}
self.tokenise(TokenType::Eof, '\0'.into());
}
fn lex_flustered_emoticon(&mut self) -> TokenType {
match self.peek() {
Some('x') => self.lex_flustered_end(TokenType::FlusteredX, '<'),
Some('o') => self.lex_flustered_end(TokenType::FlusteredO, '<'),
Some('w') => self.lex_flustered_end(TokenType::FlusteredW, '<'),
Some('~') => self.lex_flustered_end(TokenType::FlusteredTilde, '<'),
Some('.') => self.lex_flustered_end(TokenType::FlusteredDot, '<'),
Some(':') => {
self.next();
self.lex_colon_three(false)
}
c if c == Some('/') || c == Some('\\') => {
self.next();
self.lex_blush_slashes_emoticon(c.unwrap())
}
Some(_) | None => {
self.ctx.report(diagnostic!(
ErrorKind::AmbiguousUnfinishedEmoticon {
interp_title: self.ctx.rand_interp_title().into(),
petname: self.ctx.rand_petname().into(),
},
labels = [(self.byte_range(), "")]
));
TokenType::Error
}
}
}
fn lex_flustered_end(&mut self, token_type: TokenType, end: char) -> TokenType {
self.next();
if self.matches(end) {
token_type
} else {
self.ctx.report(diagnostic!(
ErrorKind::UnfinishedEmoticon {
praise_term: self.ctx.rand_praise_term().into(),
petname: self.ctx.rand_petname().into(),
char_to_add: end
},
labels = [(self.byte_range(), "")]
));
TokenType::Error
}
}
fn lex_blush_slashes_emoticon(&mut self, slash: char) -> TokenType {
fn check_double_slash(this: &mut Lexer, slash: char) -> bool {
if this.peek().is_some_and(|peek| peek != slash) {
this.ctx.report(diagnostic!(
ErrorKind::UnfinishedEmoticon {
praise_term: this.ctx.rand_praise_term().into(),
petname: this.ctx.rand_petname().into(),
char_to_add: slash
},
labels = [(this.byte_range(), "")]
));
false
} else {
this.next();
true
}
}
if !check_double_slash(self, slash) {
return TokenType::Error;
}
let mut len = 1;
while self.matches(slash) {
if !check_double_slash(self, slash) {
return TokenType::Error;
}
if len == KEYSMASH_MAX_LEN {
self.ctx.report(diagnostic!(
ErrorKind::OverlongKeysmash {
interp_title: self.ctx.rand_interp_title().into(),
petname: self.ctx.rand_petname().into()
},
labels = [(self.byte_range(), "")]
));
return TokenType::Error;
}
len += 1;
}
if self.matches('<') {
return TokenType::Blush {
double: slash == '\\',
len,
};
}
self.ctx.report(diagnostic!(
ErrorKind::UnfinishedEmoticon {
praise_term: self.ctx.rand_praise_term().into(),
petname: self.ctx.rand_petname().into(),
char_to_add: '<'
},
labels = [(self.byte_range(), "")]
));
TokenType::Error
}
fn lex_flag_emoji(&mut self) {
let diag = |this: &&mut Lexer, glyph: char| {
diagnostic!(
ErrorKind::UnexpectedToken {
petname: this.ctx.rand_petname().into(),
interp_title: this.ctx.rand_interp_title().into(),
praise_term: this.ctx.rand_praise_term().into(),
char: glyph,
},
labels = [(this.range(), "")]
)
};
for glyph in ['\u{fe0f}', '\u{200d}'] {
if !self.matches(glyph) {
self.ctx.report(diag(&self, glyph));
return self.tokenise(TokenType::Error, self.lexeme());
}
}
if self.matches('\u{1f308}') {
return self.tokenise(TokenType::PrintAnsi, self.lexeme());
}
for glyph in ['\u{26a7}', '\u{fe0f}'] {
if !self.matches(glyph) {
self.ctx.report(diag(&self, glyph));
return self.tokenise(TokenType::Error, self.lexeme());
}
}
while self.peek().is_some_and(|peek| peek != '\n') {
self.has_comments = true;
self.next();
}
}
fn lex_heavy_flustered_emoticon(&mut self, start: char) -> TokenType {
match start {
'@' => {
if self.check('~') {
self.lex_flustered_end(TokenType::HeavyFlusteredAt, '@')
} else {
self.ctx.report(diagnostic!(
ErrorKind::UnfinishedEmoticon {
praise_term: self.ctx.rand_praise_term().into(),
petname: self.ctx.rand_petname().into(),
char_to_add: '3'
},
labels = [(self.byte_range(), "")]
));
return TokenType::Error;
}
}
'O' => {
if self.check('~') {
self.lex_flustered_end(TokenType::HeavyFlusteredO, 'O')
} else {
self.lex_keysmash(start)
}
}
_ => {
self.ctx.report(diagnostic!(
ErrorKind::Bug {
interp_title: self.ctx.rand_interp_title().into(),
praise_term: self.ctx.rand_praise_term().into(),
},
labels = [(self.byte_range(), "")]
));
TokenType::Error
}
}
}
fn lex_happy_emoticon(&mut self) -> TokenType {
match self.peek() {
Some('x') => self.lex_flustered_end(TokenType::HappyX, '^'),
Some('o') => self.lex_flustered_end(TokenType::HappyO, '^'),
Some('w') => self.lex_flustered_end(TokenType::HappyW, '^'),
Some(_) | None => {
self.ctx.report(diagnostic!(
ErrorKind::AmbiguousUnfinishedEmoticon {
interp_title: self.ctx.rand_interp_title().into(),
petname: self.ctx.rand_petname().into(),
},
labels = [(self.byte_range(), "")]
));
TokenType::Error
}
}
}
fn lex_colon_three(&mut self, add: bool) -> TokenType {
let mut len = 0;
while self.matches('3') {
if len == KEYSMASH_MAX_LEN {
self.ctx.report(diagnostic!(
ErrorKind::OverlongKeysmash {
interp_title: self.ctx.rand_interp_title().into(),
petname: self.ctx.rand_petname().into()
},
labels = [(self.byte_range(), "")]
));
return TokenType::Error;
}
len += 1;
}
if len == 0 {
self.ctx.report(diagnostic!(
ErrorKind::UnfinishedEmoticon {
praise_term: self.ctx.rand_praise_term().into(),
petname: self.ctx.rand_petname().into(),
char_to_add: '3'
},
labels = [(self.byte_range(), "")]
));
return TokenType::Error;
}
TokenType::ColonThree { add, len }
}
fn lex_keysmash(&mut self, start: char) -> TokenType {
let lowercase = start.is_lowercase();
if !self.is_valid_keysmash_char(start) {
self.ctx.report(diagnostic!(
ErrorKind::UnexpectedToken {
petname: self.ctx.rand_petname().into(),
interp_title: self.ctx.rand_interp_title().into(),
praise_term: self.ctx.rand_praise_term().into(),
char: start
},
labels = [(self.byte_range(), "")]
));
return TokenType::Error;
}
let mut len = 1;
while let Some(char) = self.peek() {
if !self.is_valid_keysmash_char(char) {
break;
}
if lowercase != char.is_lowercase() {
return TokenType::Keysmash { lowercase, len };
}
if len == KEYSMASH_MAX_LEN {
self.ctx.report(diagnostic!(
ErrorKind::OverlongKeysmash {
interp_title: self.ctx.rand_interp_title().into(),
petname: self.ctx.rand_petname().into()
},
labels = [(self.byte_range(), "")]
));
return TokenType::Error;
}
len += 1;
self.next();
continue;
}
let lexeme = self.lexeme();
if self.ctx.env_vars.print_keywords.contains(&lexeme) {
return TokenType::Print {
utf: self.matches('~'),
};
}
if self.ctx.env_vars.interp_titles.contains(&lexeme) {
TokenType::InterpTitle {
pretty: self.matches('~'),
}
} else {
TokenType::Keysmash { lowercase, len }
}
}
}