#[derive(Debug, Clone)]
pub enum Symbol {
Identifier(String),
Assignment,
Integer(i64),
Float(f64),
String(String),
Boolean(bool),
Null,
Single(char),
}
#[derive(Debug)]
pub enum LexerError {
MissingNumberBaseIdentifier,
FloatBaseIsNot10,
NumberParseError(String),
AssignmentIsNotAssignment, ExpectedSingleChar,
InvalidStringInitialization,
ExpectedStringChar,
ExpectedEscapedChar,
}
impl std::fmt::Display for LexerError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{self:?}")
}
}
impl std::error::Error for LexerError {
fn description(&self) -> &str {
use LexerError::*;
match self {
MissingNumberBaseIdentifier => "Missing number base identifier",
FloatBaseIsNot10 => "Float base is not 10",
NumberParseError(_) => "Number parse error",
AssignmentIsNotAssignment => "Assignment is not assignment",
ExpectedSingleChar => "Expected single char",
InvalidStringInitialization => "Invalid string initialization",
ExpectedStringChar => "Expected string char",
ExpectedEscapedChar => "Expected escaped char",
}
}
}
#[derive(Debug, Default)]
pub struct LexerOptions {
pub human_boolean: bool
}
#[derive(Debug, Default)]
pub struct Lexer {
buffer: Vec<u8>,
cursor: isize,
symbols: Vec<Symbol>,
options: LexerOptions
}
impl Lexer {
pub fn new(buffer: Vec<u8>, options: LexerOptions) -> Self {
Self {
buffer,
options,
cursor: -1,
..Default::default()
}
}
pub fn lex(&mut self) -> Result<Vec<Symbol>, LexerError> {
while let Some(char) = self.peek() {
let symbol = match char {
' ' | '\t' | '\n' => { self.consume(); continue }
'#' => { self.comment()?; continue }
'a'..='z' | 'A'..='Z' | '_' => self.identifier()?,
'=' => self.assignment()?,
'0'..='9' | '.' | '-' => self.number()?,
'"' => self.string()?,
'(' | '[' | '{'
| ')' | ']' | '}'
| ',' => self.single()?,
_ => break,
};
self.symbols.push(symbol);
}
Ok(self.symbols.clone())
}
fn next(&mut self) -> Option<char> {
self.consume();
self.buffer.get(self.cursor as usize).map(|&byte| byte as char)
}
fn peek(&self) -> Option<char> {
self.buffer.get((self.cursor + 1) as usize).map(|&byte| byte as char)
}
fn consume(&mut self) {
self.cursor += 1;
}
fn comment(&mut self) -> Result<(), LexerError> {
while self.next().unwrap_or('\n') != '\n' {}
Ok(())
}
fn identifier(&mut self) -> Result<Symbol, LexerError> {
let mut string = String::new();
while let Some(char) = self.peek() {
if !matches!(char, 'a'..='z' | 'A'..='Z' | '0'..='9' | '_') {
break;
}
string.push(char);
self.consume();
}
Ok(self.keyword(&string).unwrap_or(Symbol::Identifier(string)))
}
fn single(&mut self) -> Result<Symbol, LexerError> {
let char = self.next().ok_or(LexerError::ExpectedSingleChar)?;
Ok(Symbol::Single(char))
}
fn keyword(&self, string: impl AsRef<str>) -> Option<Symbol> {
Some(match string.as_ref() {
"null" => Symbol::Null,
"true" => Symbol::Boolean(true),
"yes" if self.options.human_boolean => Symbol::Boolean(true),
"false" => Symbol::Boolean(false),
"no" if self.options.human_boolean => Symbol::Boolean(false),
_ => return None
})
}
fn assignment(&mut self) -> Result<Symbol, LexerError> {
if let Some(char) = self.peek() {
if char != '=' {
return Err(LexerError::AssignmentIsNotAssignment);
}
}
self.consume();
Ok(Symbol::Assignment)
}
fn number(&mut self) -> Result<Symbol, LexerError> {
let mut string = String::new();
let mut float = false;
let mut base = 10;
while let Some(char) = self.peek() {
if !matches!(char, '0'..='9' | 'a'..='f' | 'A'..='F' | '.' | '-' | '_') {
break;
}
if string.is_empty() && char == '0' {
self.consume();
if let Some(char) = self.peek() {
match char {
'x' | 'X' => base = 16,
'd' | 'D' => base = 10,
'o' | 'O' => base = 8,
'b' | 'B' => base = 2,
'0'..='9' => return Err(LexerError::MissingNumberBaseIdentifier), _ => {
return Ok(Symbol::Integer(0));
}, }
self.consume();
continue;
}
}
if char == '.' {
float = true;
if string.is_empty() {
string.push('0');
}
}
if char != '_' {
string.push(char);
}
self.consume();
}
if float {
if base != 10 {
return Err(LexerError::FloatBaseIsNot10)
}
Ok(Symbol::Float(string.parse::<f64>().map_err(|err| LexerError::NumberParseError(err.to_string()))?))
} else {
Ok(Symbol::Integer(i64::from_str_radix(&string, base).map_err(|err| LexerError::NumberParseError(err.to_string()))?))
}
}
fn string(&mut self) -> Result<Symbol, LexerError> {
let mut string = String::with_capacity(32);
if !self.next().is_some_and(|char| char == '"') {
return Err(LexerError::InvalidStringInitialization);
}
loop {
let mut char = self.next().ok_or(LexerError::ExpectedStringChar)?;
if char == '"' {
break;
}
if char == '\\' {
char = self.next().ok_or(LexerError::ExpectedEscapedChar)?;
}
string.push(char);
}
Ok(Symbol::String(string))
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn lexer_parse_comments() -> Result<(), Box<dyn std::error::Error>> {
let buffer = b"# comment at the start of the file\n# comment after a comment";
let mut lexer = Lexer::new(buffer.to_vec(), Default::default());
let symbols = lexer.lex()?;
assert_eq!(symbols.len(), 0);
Ok(())
}
#[test]
fn lexer_parse_identifiers() -> Result<(), Box<dyn std::error::Error>> {
let buffer = b"some r4nd0m _words i";
let mut lexer = Lexer::new(buffer.to_vec(), Default::default());
let symbols = lexer.lex()?;
assert_eq!(symbols.len(), 4);
assert!(matches!(symbols[0], Symbol::Identifier(ref string) if string == "some"));
assert!(matches!(symbols[1], Symbol::Identifier(ref string) if string == "r4nd0m"));
assert!(matches!(symbols[2], Symbol::Identifier(ref string) if string == "_words"));
assert!(matches!(symbols[3], Symbol::Identifier(ref string) if string == "i"));
Ok(())
}
#[test]
fn lexer_parse_assignments() -> Result<(), Box<dyn std::error::Error>> {
let buffer = b"= == =";
let mut lexer = Lexer::new(buffer.to_vec(), Default::default());
let symbols = lexer.lex()?;
assert_eq!(symbols.len(), 4);
for symbol in symbols {
assert!(matches!(symbol, Symbol::Assignment));
}
Ok(())
}
#[test]
fn lexer_parse_numbers() -> Result<(), Box<dyn std::error::Error>> {
let buffer = b"-234 423.54 10.0 .54 90_230_000 0xcF .wut 0";
let mut lexer = Lexer::new(buffer.to_vec(), Default::default());
let symbols = lexer.lex()?;
assert_eq!(symbols.len(), 9);
assert!(matches!(symbols[0], Symbol::Integer(-234)));
assert!(matches!(symbols[1], Symbol::Float(number) if number == 423.54));
assert!(matches!(symbols[2], Symbol::Float(number) if number == 10.0));
assert!(matches!(symbols[3], Symbol::Float(number) if number == 0.54));
assert!(matches!(symbols[4], Symbol::Integer(90_230_000)));
assert!(matches!(symbols[5], Symbol::Integer(0xCF)));
assert!(matches!(symbols[6], Symbol::Float(number) if number == 0.0));
assert!(matches!(symbols[7], Symbol::Identifier(ref string) if string == "wut"));
assert!(matches!(symbols[8], Symbol::Integer(0)));
Ok(())
}
#[test]
fn lexer_parse_strings() -> Result<(), Box<dyn std::error::Error>> {
let buffer = br#""some string" "and one with escaped \" quotes""#;
let mut lexer = Lexer::new(buffer.to_vec(), Default::default());
let symbols = lexer.lex()?;
assert_eq!(symbols.len(), 2);
assert!(matches!(symbols[0], Symbol::String(ref string) if string == "some string"));
assert!(matches!(symbols[1], Symbol::String(ref string) if string == "and one with escaped \" quotes"));
Ok(())
}
#[test]
fn lexer_parse_booleans() -> Result<(), Box<dyn std::error::Error>> {
let buffer = b"true false";
let mut lexer = Lexer::new(buffer.to_vec(), Default::default());
let symbols = lexer.lex()?;
assert_eq!(symbols.len(), 2);
assert!(matches!(symbols[0], Symbol::Boolean(true)));
assert!(matches!(symbols[1], Symbol::Boolean(false)));
Ok(())
}
#[test]
fn lexer_parse_null() -> Result<(), Box<dyn std::error::Error>> {
let buffer = b"null";
let mut lexer = Lexer::new(buffer.to_vec(), Default::default());
let symbols = lexer.lex()?;
assert_eq!(symbols.len(), 1);
assert!(matches!(symbols[0], Symbol::Null));
Ok(())
}
#[test]
fn lexer_parse_scopes() -> Result<(), Box<dyn std::error::Error>> {
let buffer = b"[([{)babadoue)}{";
let mut lexer = Lexer::new(buffer.to_vec(), Default::default());
let symbols = lexer.lex()?;
assert_eq!(symbols.len(), 9);
assert!(matches!(symbols[0], Symbol::Single('[')));
assert!(matches!(symbols[1], Symbol::Single('(')));
assert!(matches!(symbols[2], Symbol::Single('[')));
assert!(matches!(symbols[3], Symbol::Single('{')));
assert!(matches!(symbols[4], Symbol::Single(')')));
assert!(matches!(symbols[5], Symbol::Identifier(ref string) if string == "babadoue"));
assert!(matches!(symbols[6], Symbol::Single(')')));
assert!(matches!(symbols[7], Symbol::Single('}')));
assert!(matches!(symbols[8], Symbol::Single('{')));
Ok(())
}
#[test]
fn lexer_parse_human_booleans() -> Result<(), Box<dyn std::error::Error>> {
let buffer = b"yes no true false good bad";
let mut lexer = Lexer::new(buffer.to_vec(), LexerOptions { human_boolean: true });
let symbols = lexer.lex()?;
assert_eq!(symbols.len(), 6);
assert!(matches!(symbols[0], Symbol::Boolean(true)));
assert!(matches!(symbols[1], Symbol::Boolean(false)));
assert!(matches!(symbols[2], Symbol::Boolean(true)));
assert!(matches!(symbols[3], Symbol::Boolean(false)));
assert!(matches!(symbols[4], Symbol::Identifier(_)));
assert!(matches!(symbols[5], Symbol::Identifier(_)));
Ok(())
}
}