use std::cmp::Ordering;
use std::collections::HashMap;
use std::ops::Range;
use super::error::{Error, ErrorKind};
use super::error_strategy::error_listener::ErrorListener;
use super::lexer_rule::LexerRule;
use super::position::Position;
use super::token::Token;
pub trait Lexer {
fn iter(&self) -> TokenIter ;
fn get_all_on_channel_tokens(&self, channel: usize) -> Vec<Token> {
self.iter().filter(|token| token.channel == channel).collect::<Vec<_>>()
}
fn get_all_tokens(&self) -> Vec<Token> {
self.iter().collect::<Vec<_>>()
}
fn scan_all_tokens_and_group_by_channel(&mut self) -> HashMap<usize, Vec<Token>> {
let mut ret: HashMap<usize, Vec<Token>> = HashMap::new();
for token in self.iter() {
if ret.contains_key(&token.channel) {
ret.get_mut(&token.channel).unwrap().push(token);
} else {
ret.insert(token.channel, vec![token]);
}
}
ret
}
}
pub struct TokenIter<'a> {
pub input: &'a str, pub rules: &'a [LexerRule],
pub error_listeners: &'a [Box<dyn ErrorListener>],
pub ranges: Vec<Range<usize>>,
pub cursor: usize, pub token_index: usize, }
impl<'a> TokenIter<'a> {
pub fn get_current_position(&self) -> Position {
self.get_position_from_char_index(self.cursor)
}
fn get_position_from_char_index(&self, char_index: usize) -> Position {
let line = self.ranges.binary_search_by(|range| {
if range.end <= char_index {
Ordering::Less
} else if range.start > char_index {
Ordering::Greater
} else {
Ordering::Equal
}
}).unwrap();
let range = self.ranges[line].clone();
Position::new(line, char_index - range.start)
}
pub fn lexer_match(&mut self) -> Result<Token, Error> {
if self.cursor >= self.input.len() {
return Err(Error::new(ErrorKind::LexerScanOverflow, "LexerScanOverflow",
self.get_position_from_char_index(self.cursor),
self.get_position_from_char_index(self.cursor)));
}
let mut len = 0;
let mut start = self.input.len();
let mut stop = start;
let mut meta: Option<LexerRule> = None;
for lexer_meta in self.rules.iter() {
if ! lexer_meta.rule.is_match_at(self.input, self.cursor) { continue; }
let result = lexer_meta.rule.find_at(self.input, self.cursor) ;
if let Some(result) = result {
if result.start() < start || result.start() == start && result.end() - result.start() > len {
meta = Some(lexer_meta.clone());
start = result.start();
stop = result.end();
len = result.end() - result.start();
}
}
}
if let None = meta {
return Err(Error::new(ErrorKind::LexerNoMatch, "",
self.get_position_from_char_index(self.cursor),
self.get_position_from_char_index(self.cursor)));
}
if start != self.cursor {
for listener in self.error_listeners.iter() {
listener.syntax_error();
}
todo!()
}
let text = String::from(&self.input[start..stop]);
let meta = meta.unwrap();
let token = Token::new(meta.token_type, &meta.token_name, &text,
self.get_position_from_char_index(start),
self.get_position_from_char_index(stop), self.token_index,
meta.channel,
self.cursor,
self.cursor + len);
self.cursor = stop;
if meta.skip {
return self.lexer_match();
}
self.token_index += 1;
return Ok(token);
}
pub fn reset(&mut self) {
self.cursor = 0;
self.token_index = 1;
}
pub fn new(input: &'a str, rules: &'a [LexerRule], error_listeners: &'a [Box<dyn ErrorListener>]) -> Self {
let mut st = 0;
let ranges = input.split("\n").map(|f| {
let ed = st + f.len() + 1; let ret = st..ed;
st = ed;
ret
}).collect::<Vec<_>>();
Self {
input, rules, error_listeners, cursor: 0, token_index: 1,
ranges,
}
}
}
impl Iterator for TokenIter<'_> {
type Item = Token;
fn next(&mut self) -> Option<Self::Item> {
match self.lexer_match() {
Ok(token) => Some(token),
Err(_) => None,
}
}
}