use std::{iter::Peekable, str::Chars};
#[derive(Clone, Debug)]
pub struct Position {
pub column: usize,
pub line: usize,
pub index: usize,
}
impl Position {
pub fn new() -> Position {
Position {
column: 1,
line: 1,
index: 0,
}
}
}
#[derive(Clone, Debug)]
pub struct Spanned<T> {
pub item: T,
pub start: Position,
pub end: Position,
}
impl<T> Spanned<T> {
pub fn new(item: T, start: Position, end: Position) -> Spanned<T> {
Spanned { item, start, end }
}
}
#[derive(Clone, PartialEq, Eq, Hash, Debug)]
pub enum Token {
Symbol(String),
OpenParen,
CloseParen,
Comma,
Dot,
Semi,
Arrow,
Question,
Eq,
NEq,
Gt,
GtEq,
Lt,
LtEq,
Plus,
Minus,
Star,
Slash,
Mod,
Not,
And,
Or,
Int(String),
String(String),
Null,
}
pub struct Lexer<'a> {
input: &'a str,
chars: Peekable<Chars<'a>>,
position: Position,
}
impl<'a> Lexer<'a> {
pub fn new(input: &'a str) -> Lexer<'a> {
Lexer {
input,
chars: input.chars().peekable(),
position: Position::new(),
}
}
fn advance(&mut self) -> char {
let c = self.chars.next().unwrap();
if c == '\n' {
self.position.column = 1;
self.position.line += 1;
self.position.index += 1;
} else {
self.position.column += 1;
self.position.index += 1;
}
c
}
fn skip_ignore(&mut self) {
while let Some(c) = self.chars.peek().copied() {
match c {
' ' | '\n' | '\t' => {
self.advance();
}
_ => break,
}
}
}
}
#[derive(Clone, Debug)]
pub enum LexerError {
UnexpectedCharacter(char),
}
fn is_symbol_start(c: char) -> bool {
match c {
'a'..='z' | 'A'..='Z' | '_' => true,
_ => false,
}
}
fn is_symbol_continue(c: char) -> bool {
match c {
'a'..='z' | 'A'..='Z' | '_' | '0'..='9' => true,
_ => false,
}
}
fn is_number_start_or_continue(c: char) -> bool {
match c {
'0'..='9' => true,
_ => false,
}
}
fn descape_symbol(s: &str) -> String {
if s.starts_with('"') && s.ends_with('"') {
s[1..s.len() - 1].to_string()
} else {
s.to_string()
}
}
impl<'a> Iterator for Lexer<'a> {
type Item = Result<Spanned<Token>, Spanned<LexerError>>;
fn next(&mut self) -> Option<Self::Item> {
self.skip_ignore();
let start = self.position.clone();
match *self.chars.peek()? {
',' => {
self.advance();
Some(Ok(Spanned::new(Token::Comma, start, self.position.clone())))
}
'?' => {
self.advance();
Some(Ok(Spanned::new(
Token::Question,
start,
self.position.clone(),
)))
}
'.' => {
self.advance();
Some(Ok(Spanned::new(Token::Dot, start, self.position.clone())))
}
'=' => {
self.advance();
Some(Ok(Spanned::new(Token::Eq, start, self.position.clone())))
}
'+' => {
self.advance();
Some(Ok(Spanned::new(Token::Plus, start, self.position.clone())))
}
'-' => {
self.advance();
if let Some('>') = self.chars.peek() {
self.advance();
Some(Ok(Spanned::new(Token::Arrow, start, self.position.clone())))
} else {
Some(Ok(Spanned::new(Token::Minus, start, self.position.clone())))
}
}
'*' => {
self.advance();
Some(Ok(Spanned::new(Token::Star, start, self.position.clone())))
}
'/' => {
self.advance();
Some(Ok(Spanned::new(Token::Slash, start, self.position.clone())))
}
'%' => {
self.advance();
Some(Ok(Spanned::new(Token::Mod, start, self.position.clone())))
}
'!' => {
self.advance();
if let Some('=') = self.chars.peek() {
self.advance();
Some(Ok(Spanned::new(Token::NEq, start, self.position.clone())))
} else {
return Some(Err(Spanned::new(
LexerError::UnexpectedCharacter(self.advance()),
start,
self.position.clone(),
)));
}
}
'>' => {
self.advance();
if let Some('=') = self.chars.peek() {
self.advance();
Some(Ok(Spanned::new(Token::GtEq, start, self.position.clone())))
} else {
Some(Ok(Spanned::new(Token::Gt, start, self.position.clone())))
}
}
'<' => {
self.advance();
if let Some('=') = self.chars.peek() {
self.advance();
Some(Ok(Spanned::new(Token::LtEq, start, self.position.clone())))
} else {
Some(Ok(Spanned::new(Token::Lt, start, self.position.clone())))
}
}
'(' => {
self.advance();
Some(Ok(Spanned::new(
Token::OpenParen,
start,
self.position.clone(),
)))
}
')' => {
self.advance();
Some(Ok(Spanned::new(
Token::CloseParen,
start,
self.position.clone(),
)))
}
';' => {
self.advance();
Some(Ok(Spanned::new(Token::Semi, start, self.position.clone())))
}
c if is_symbol_start(c) => {
self.advance();
while let Some(c) = self.chars.peek().copied() {
if is_symbol_continue(c) {
self.advance();
} else {
break;
}
}
let s = &self.input[start.index..self.position.index];
if s == "NULL" {
Some(Ok(Spanned::new(Token::Null, start, self.position.clone())))
} else if s == "NOT" {
Some(Ok(Spanned::new(Token::Not, start, self.position.clone())))
} else if s == "AND" {
Some(Ok(Spanned::new(Token::And, start, self.position.clone())))
} else if s == "OR" {
Some(Ok(Spanned::new(Token::Or, start, self.position.clone())))
} else {
Some(Ok(Spanned::new(
Token::Symbol(descape_symbol(s)),
start,
self.position.clone(),
)))
}
}
'\'' => {
self.advance();
while let Some(c) = self.chars.peek().copied() {
match c {
'\'' => {
self.advance();
if self.chars.peek().copied() == Some('\'') {
self.advance();
} else {
break;
}
}
_ => {
self.advance();
}
}
}
Some(Ok(Spanned::new(
Token::String(self.input[start.index..self.position.index].to_string()),
start,
self.position.clone(),
)))
}
'"' => {
self.advance();
while let Some(c) = self.chars.peek().copied() {
match c {
'"' => {
self.advance();
break;
}
_ => {
self.advance();
}
}
}
Some(Ok(Spanned::new(
Token::Symbol(descape_symbol(
&self.input[start.index..self.position.index],
)),
start,
self.position.clone(),
)))
}
c if is_number_start_or_continue(c) => {
self.advance();
while let Some(c) = self.chars.peek().copied() {
if is_symbol_continue(c) {
self.advance();
} else {
break;
}
}
Some(Ok(Spanned::new(
Token::Int(self.input[start.index..self.position.index].to_string()),
start,
self.position.clone(),
)))
}
c => {
self.advance();
return Some(Err(Spanned::new(
LexerError::UnexpectedCharacter(c),
start,
self.position.clone(),
)));
}
}
}
}