use crate::errors::predule::LexingError;
use crate::errors::RRDBError;
use crate::lexer::predule::{OperatorToken, Token};
use crate::logger::predule::Logger;
#[derive(Debug)]
pub struct Tokenizer {
buffer: Vec<char>,
buffer_index: usize,
last_char: char,
}
impl Tokenizer {
pub fn new(text: String) -> Self {
Logger::info(format!("SQL echo: {:?}", text));
Self {
last_char: ' ',
buffer: text.chars().collect(),
buffer_index: 0,
}
}
pub fn is_whitespace(&self) -> bool {
self.last_char == ' ' || self.last_char == '\n' || self.last_char == '\t'
}
pub fn is_digit(&self) -> bool {
self.last_char.is_ascii_digit()
}
pub fn is_alphabet(&self) -> bool {
self.last_char.is_alphabetic()
}
pub fn is_alphabet_or_number(&self) -> bool {
self.last_char.is_alphanumeric()
}
pub fn is_underscore(&self) -> bool {
self.last_char == '_'
}
pub fn is_special_character(&self) -> bool {
['+', '-', '*', '/', ',', '>', '<', '=', '!', '\\', '@'].contains(&self.last_char)
}
pub fn is_quote(&self) -> bool {
['\'', '"'].contains(&self.last_char)
}
pub fn is_semicolon(&self) -> bool {
self.last_char == ';'
}
pub fn is_dot(&self) -> bool {
self.last_char == '.'
}
pub fn is_backtick(&self) -> bool {
self.last_char == '`'
}
pub fn is_parentheses(&self) -> bool {
self.last_char == '(' || self.last_char == ')'
}
pub fn is_eof(&self) -> bool {
self.buffer_index >= self.buffer.len()
}
pub fn read_char(&mut self) {
if self.buffer_index >= self.buffer.len() {
self.last_char = ' ';
} else {
self.last_char = self.buffer[self.buffer_index];
self.buffer_index += 1;
}
}
pub fn unread_char(&mut self) {
if self.buffer_index != 0 {
self.buffer_index -= 1;
self.last_char = self.buffer[self.buffer_index];
}
}
pub fn get_token(&mut self) -> Result<Token, RRDBError> {
while self.is_whitespace() && !self.is_eof() {
self.read_char();
}
let token = if self.is_alphabet() || self.is_underscore() {
let mut identifier = vec![self.last_char];
self.read_char();
while self.is_alphabet_or_number() || self.is_underscore() {
identifier.push(self.last_char);
self.read_char();
}
let identifier: String = identifier.into_iter().collect::<String>();
let token = match identifier.to_uppercase().as_str() {
"SELECT" => Token::Select,
"FROM" => Token::From,
"WHERE" => Token::Where,
"AS" => Token::As,
"ORDER" => Token::Order,
"BY" => Token::By,
"ASC" => Token::Asc,
"DESC" => Token::Desc,
"GROUP" => Token::Group,
"HAVING" => Token::Having,
"LIMIT" => Token::Limit,
"OFFSET" => Token::Offset,
"INSERT" => Token::Insert,
"INTO" => Token::Into,
"VALUES" => Token::Values,
"UPDATE" => Token::Update,
"SET" => Token::Set,
"DELETE" => Token::Delete,
"JOIN" => Token::Join,
"INNER" => Token::Inner,
"LEFT" => Token::Left,
"RIGHT" => Token::Right,
"FULL" => Token::Full,
"OUTER" => Token::Outer,
"CREATE" => Token::Create,
"ALTER" => Token::Alter,
"DROP" => Token::Drop,
"DATABASE" => Token::Database,
"TABLE" => Token::Table,
"COLUMN" => Token::Column,
"COMMENT" => Token::Comment,
"PRIMARY" => Token::Primary,
"FOREIGN" => Token::Foreign,
"KEY" => Token::Key,
"ADD" => Token::Add,
"RENAME" => Token::Rename,
"TO" => Token::To,
"SHOW" => Token::Show,
"DATABASES" => Token::Databases,
"TABLES" => Token::Tables,
"AND" => Token::And,
"OR" => Token::Or,
"NOT" => Token::Not,
"BETWEEN" => Token::Between,
"LIKE" => Token::Like,
"IN" => Token::In,
"IS" => Token::Is,
"TRUE" => Token::Boolean(true),
"FALSE" => Token::Boolean(false),
"NULL" => Token::Null,
"DEFAULT" => Token::Default,
"IF" => Token::If,
"EXISTS" => Token::Exists,
"ON" => Token::On,
"USE" => Token::Use,
"DATA" => Token::Data,
"TYPE" => Token::Type,
"NULLS" => Token::Nulls,
"FIRST" => Token::First,
"LAST" => Token::Last,
"BEGIN" => Token::Begin,
"TRANSACTION" => Token::Transaction,
"COMMIT" => Token::Commit,
"ROLLBACK" => Token::Rollback,
_ => Token::Identifier(identifier),
};
return Ok(token);
}
else if self.is_digit() {
let mut number_string = vec![self.last_char];
loop {
self.read_char();
if self.is_digit() || self.is_dot() {
number_string.push(self.last_char);
continue;
} else if self.is_eof() {
break;
} else {
self.unread_char();
break;
}
}
let number_string: String =
number_string.into_iter().collect::<String>().to_uppercase();
if number_string.contains('.') {
let number = number_string.parse::<f64>();
match number {
Ok(number) => Token::Float(number),
Err(_) => {
return Err(LexingError::wrap(format!(
"invalid floating point number format: {}",
number_string
)))
}
}
} else {
let number = number_string.parse::<i64>();
match number {
Ok(number) => Token::Integer(number),
Err(_) => {
return Err(LexingError::wrap(format!(
"invalid integer number format: {}",
number_string
)))
}
}
}
}
else if self.is_special_character() {
match self.last_char {
',' => Token::Comma,
'\\' => Token::Backslash,
'-' => {
self.read_char();
if self.last_char == '-' {
let mut comment = vec![];
while !self.is_eof() {
self.read_char();
if self.last_char == '\n' {
break;
} else {
comment.push(self.last_char);
}
}
let comment: String = comment.into_iter().collect();
Token::CodeComment(comment)
} else {
self.unread_char();
Token::Operator(OperatorToken::Minus)
}
}
'/' => {
self.read_char();
if self.last_char == '*' {
let mut comment = vec![];
self.read_char();
while !self.is_eof() {
if self.last_char == '*' {
self.read_char();
if self.last_char == '/' {
break;
}
} else {
comment.push(self.last_char);
}
self.read_char();
}
let comment: String = comment.into_iter().collect();
Token::CodeComment(comment)
} else {
self.unread_char();
Token::Operator(OperatorToken::Slash)
}
}
'+' => Token::Operator(OperatorToken::Plus),
'*' => Token::Operator(OperatorToken::Asterisk),
'!' => Token::Operator(OperatorToken::Not), '=' => Token::Operator(OperatorToken::Eq),
'<' => Token::Operator(OperatorToken::Lt), '>' => Token::Operator(OperatorToken::Gt), _ => {
return Err(LexingError::wrap(format!(
"unexpected operator: {:?}",
self.last_char
)))
}
}
}
else if self.is_quote() {
if self.last_char == '"' {
let mut identifier = vec![];
self.read_char();
while self.last_char != '"' {
identifier.push(self.last_char);
self.read_char();
}
let identifier: String = identifier.into_iter().collect::<String>();
Token::Identifier(identifier)
} else {
let mut string = vec![];
self.read_char();
while !self.is_eof() {
if self.last_char == '\'' {
self.read_char();
if self.last_char == '\'' {
string.push(self.last_char);
} else {
self.unread_char();
break;
}
} else {
string.push(self.last_char);
}
self.read_char();
}
let string: String = string.into_iter().collect::<String>();
Token::String(string)
}
} else if self.is_backtick() {
let mut string = vec![];
self.read_char();
while !self.is_eof() {
if self.last_char == '`' {
self.read_char();
if self.last_char == '`' {
string.push(self.last_char);
} else {
self.unread_char();
break;
}
} else {
string.push(self.last_char);
}
self.read_char();
}
let string: String = string.into_iter().collect::<String>();
Token::Identifier(string)
}
else if self.is_semicolon() {
Token::SemiColon
}
else if self.is_dot() {
Token::Period
}
else if self.is_parentheses() {
if self.last_char == '(' {
Token::LeftParentheses
} else {
Token::RightParentheses
}
}
else if self.is_eof() {
Token::EOF
} else {
return Err(LexingError::wrap(format!(
"unexpected character: {:?}",
self.last_char
)));
};
self.last_char = ' ';
Ok(token)
}
pub fn string_to_tokens(text: String) -> Result<Vec<Token>, RRDBError> {
let mut tokenizer = Tokenizer::new(text);
let mut tokens = vec![];
while !tokenizer.is_eof() {
tokens.push(tokenizer.get_token()?);
}
Ok(tokens)
}
}