#![allow(dead_code)]
#![allow(unused_variables)]
#![allow(non_snake_case)]
#![allow(non_camel_case_types)]
#![allow(unused_parens)]
#![allow(unused_mut)]
#![allow(unused_imports)]
use std::io::{Read,Error,BufRead,BufReader,Lines,Result};
use std::cell::{RefCell,Ref,RefMut};
use std::rc::Rc;
use std::fs::File;
use std::collections::{HashMap,HashSet};
use crate::Token::*;
// hand-build a lexer
#[derive(Clone,PartialEq,Debug)]
pub enum Token
{
Integer(i64),
Float(f64),
Symbol(String),
Alphanum(String),
Keyword(String),
Stringlit(String),
Verbatim(String),
NewLine,
Nothing,
Error(&'static str),
}
fn isdelim(c:char) -> bool
{
c=='(' || c=='[' || c=='{' || c==')' || c==']' || c=='}'
}
// returns token and next index
pub fn next_token(s:&str) -> (Token, usize)
{
//let st = s.trim_start(); // assume s already trimmed
if s.len()<1 {return (NewLine,0);}
let first = s.chars().next().unwrap();
let mut index = 0; //s.len() - st.len();
if first.is_alphabetic() || first=='_' {
index=match_alphanum(s);
return (Alphanum(String::from(&s[0..index])),index);
}//alphanumeric
else if first=='.' || first.is_ascii_digit() { //possible number
index = match_num(s);
if index>0 {
if let Ok(m) = s[0..index].parse::<i64>() {return (Integer(m),index);}
else if let Ok(n) = s[0..index].parse::<f64>()
{return (Float(n),index);}
}
else {return (Symbol(first.to_string()),1);}
}
else if first=='\"' {
index = match_strlit(s);
if index>0
{return (Stringlit(String::from(&s[0..index])),index);} //includes ""
else {return (Verbatim(s.to_string()),s.len());}
}
else {
index = match_symbol(s);
if index>0 {return (Symbol(String::from(&s[0..index])),index);}
}
(Verbatim(s.to_string()),s.len()) // default
}
///// match token type, returns index of where token ended plus one.
pub fn match_alphanum(s:&str) -> usize
{
if s.len()<1 {return 0;}
let mut chars = s.chars();
let mut next = chars.next().unwrap();
if !(next.is_alphabetic() || next=='_') {return 0;}
let mut index = 1;
let mut stop=false;
while !stop && index<s.len()
{
next = chars.next().unwrap();
if next.is_alphanumeric() || next=='_' { index+=1; }
else {stop=true;}
}
return index;
}//match_alphanum
pub fn match_num(s:&str) -> usize // could be integer or float
{
let mut chars = s.chars();
let mut index = 0;
let mut stop = false;
let mut foundpoint = false;
let mut founddigit = false;
while !stop && index<s.len()
{
let mut next = chars.next().unwrap();
match (next, foundpoint) {
('.', false) => { foundpoint=true; index+=1},
('.', true) => {stop=true; index=0}, // match failed (2 .'s)
(x,_) if x.is_ascii_digit() => { founddigit=true; index+=1},
_ => {stop=true;}
}//match
}
if founddigit {index} else {0}
}
pub fn match_strlit(s:&str) -> usize
{
if s.len()<1 {return 0;}
let mut chars = s.chars();
let mut index = 0;
let mut stop = false;
let mut next = chars.next().unwrap();;
if next !='\"' {return 0;} else { index+= 1; } // will only be called if...
while !stop && index<s.len()
{
next = chars.next().unwrap();
index += 1;
if next=='\\' && index<s.len() {
next=chars.next().unwrap();
index += 1;
} //skip escape char
else if next=='\"' { stop=true; }
//else {index+=1; }
}//while
if stop {index} else {0}
} // the stringlit will include the enclosing ""'s
pub fn match_symbol(s:&str) -> usize
{
if s.len()<1 {return 0;}
let mut chars = s.chars();
let mut c = chars.next().unwrap();
if isdelim(c) {return 1;}
if c.is_ascii_digit() || c.is_alphanumeric() || c=='_' || c=='\"' || c=='.' {return 0;}
let mut index = 1;
let mut stop = false;
while !stop && index<s.len()
{
c = chars.next().unwrap();
if !c.is_whitespace() && !c.is_ascii_digit() && !c.is_alphanumeric() && c!='_' && c!='\"' && c!='.'
{index+=1;} else {stop=true;}
}
return index;
}
///////////////////////////////////
pub struct Str_tokenizer<'t>
{
slice : &'t str,
//keywords: Option<&'t HashSet<String>>,
line_comment : char,
}
impl<'t> Str_tokenizer<'t>
{
pub fn new(s:&'t str) -> Str_tokenizer<'t>
{
Str_tokenizer {
slice : s.trim_start(),
//keywords: None,
line_comment: '#', // use a whitespace for no linecomment
}
}
pub fn rest(&self) -> &'t str
{ self.slice }
pub fn no_line_comment(&mut self) {self.line_comment=' ';}
pub fn set_line_comment(&mut self, c:char) {self.line_comment='c';}
pub fn reset<'u:'t>(&mut self, s:&'u str)
{
self.slice = s;
}
}//impl Str_tokenizer
impl<'t> Iterator for Str_tokenizer<'t>
{
type Item = Token;
fn next(&mut self) -> Option<Token>
{
if self.slice.len()<1 {None}
else {
let (tok,ind) = next_token(self.slice);
let mut return_value = None;
match tok {
Symbol(s) if s.chars().next().unwrap()==self.line_comment => {
return_value = Some(Verbatim(self.slice.to_owned()));
self.slice = "";
},
/*
Alphanum(a) => {
if let Some(hsref) = self.keywords {
if hsref.contains(&a) {return_value = Some(Keyword(a));}
}
},
*/
_ => { return_value = Some(tok); },
};
if self.slice.len()>0 {self.slice = &self.slice[ind..].trim_start();}
return_value
}
}//next
}//impl Iterator for Str_tokenizer
//////////////////////////////////////////////////////////////////
//////////////// File_tokenizer //////////////////////////////////
//////////////////////////////////////////////////////////////////
#[derive(PartialEq,Eq,Copy,Clone,Debug)]
enum Mode {
normal,
comment(usize), // usize keeps starting linenum of comment
stringlit(usize), // for error reporting
}
fn iscomment(m:&Mode) -> bool
{ if let Mode::comment(n) = m {true} else {false} }
fn isstringlit(m:&Mode) -> bool
{ if let Mode::stringlit(n) = m {true} else {false} }
///////////////////////////
pub struct File_tokenizer
{
pub linenum: usize,
pub column: usize,
line : Rc<RefCell<String>>,
reader : BufReader<File>,
mode : Mode, //
current_string : String,
pub keywords: HashSet<String>,
pub singletons: HashSet<char>, // singleton symbols
pub line_comment : String,
pub keep_comments : bool,
pub begin_comment : String,
pub end_comment : String,
pub keep_nothing : bool,
pub keep_newline : bool,
}//File_tokenizer
impl File_tokenizer
{
pub fn new(filename:&str) -> File_tokenizer
{
let reader1 = match File::open(filename) {
Ok(fi) => BufReader::new(fi),
_ => {panic!("File {} not found",filename)},
};
let mut singlesyms = HashSet::<char>::new();
let mut kwhash = HashSet::<String>::new();
for c in "[](){}".chars() {singlesyms.insert(c);}
File_tokenizer {
linenum: 0,
column: 0,
line: Rc::new(RefCell::new(String::from(""))),
reader : reader1,
keywords: kwhash,
singletons: singlesyms,
line_comment: String::from("//"),
keep_comments : false,
keep_nothing : false,
keep_newline : false,
mode : Mode::normal,
current_string : String::from(""),
begin_comment : String::from("/*"),
end_comment : String::from("*/"),
}
}//new
pub fn add_keywords(&mut self, kws:&str)
{
let ki = kws.split_whitespace();
for kw in ki { self.keywords.insert(kw.trim().to_owned());}
}
pub fn add_singleton(&mut self, singles:&str)
{
for c in singles.chars() {self.singletons.insert(c);}
}
pub fn no_line_comment(&mut self) {self.line_comment=String::from("");}
pub fn set_line_comment(&mut self, c:&str)
{self.line_comment=String::from(c);}
// move some match procedures here INSIDE File_tokenizer ***
// returns token and next index
pub fn next_token(&mut self, s:&str) -> (Token, usize)
{
//let st = s.trim_start(); // assume s already trimmed
if s.len()<1 {return (NewLine,0);}
let first = s.chars().next().unwrap();
let mut index = 0;
if (s.len()>1 && &s[0..2]==&self.begin_comment[..] && self.mode==Mode::normal) || iscomment(&self.mode) {
if !iscomment(&self.mode) {self.mode = Mode::comment(self.linenum);}
//println!("IN COMMENT MODE, line {}", self.linenum);
match s.find(&self.end_comment) {
Some(index) => {
self.mode=Mode::normal;
let mut ret=std::mem::replace(&mut self.current_string,String::from(""));
if self.keep_comments {
ret.push_str(&s[0..index+self.end_comment.len()]);
return (Verbatim(ret), index+self.end_comment.len());
}
else { return (Nothing,index+self.end_comment.len()); }
},
None => {
self.current_string.push_str(s);
return (Nothing,s.len());
},
}//match
}//comment mode
else
if first=='\"' || isstringlit(&self.mode) {
if !isstringlit(&self.mode) {self.mode = Mode::stringlit(self.linenum);}
index = self.match_strlit(s);
if index>0 { // found closing quote
let mut ret=std::mem::replace(&mut self.current_string,String::from(""));
ret.push_str(&s[0..index]);
self.mode = Mode::normal;
return (Stringlit(ret),index);
}
else {
self.current_string.push_str(s);
return (Nothing,s.len());
}
}
else if first.is_alphabetic() || first=='_' {
index=match_alphanum(s);
return (Alphanum(String::from(&s[0..index])),index);
}//alphanumeric
else if first=='.' || first.is_ascii_digit() { //possible number
index = match_num(s);
if index>0 {
if let Ok(m) = s[0..index].parse::<i64>() {return (Integer(m),index);}
else if let Ok(n) = s[0..index].parse::<f64>()
{return (Float(n),index);}
}
else {return (Symbol(first.to_string()),1);}
}
else {
index = match_symbol(s);
if index>0 {return (Symbol(String::from(&s[0..index])),index);}
}
//(Verbatim(s.to_string()),s.len()) // default
(Nothing,s.len())
}//next_token inside impl File_tokenizer
pub fn match_strlit(&mut self, s:&str) -> usize
{
if s.len()<1 {return 0;}
let mut chars = s.chars();
let mut index = 0;
let mut stop = false;
let mut next = chars.next().unwrap();;
/*if next !='\"' && self.current_string.len()==0 {return 0;}
else*/ if next=='\"' && self.current_string.len()>0 { return 1; }
// above else-if is for special empty string ""
index +=1;
while !stop && index<s.len()
{
next = chars.next().unwrap();
index += 1;
if next=='\\' && index<s.len() {
next=chars.next().unwrap();
index += 1;
} //skip escape char
else if next=='\"' { stop=true; }
}//while
if stop {index} else {0}
} // the stringlit will include the enclosing ""'s - inside File_tokenizer
}//impl File_tokenizer
//////////
impl Iterator for File_tokenizer
{
type Item = Token;
fn next(&mut self) -> Option<Token>
{
let cmlen = self.line_comment.len();
let mut return_value = None;
loop // simulate do-while loop
{
// goto next line if needed:
let linerc = Rc::clone(&self.line);
let mut brline = linerc.borrow_mut();
if self.column >= brline.len() {
let mut newline = String::from("");
match self.reader.read_line(&mut newline) {
Ok(n) if n>0 => {
//self.line.replace(newline);
*brline = newline;
self.linenum +=1;
self.column=0;
},
_ => {
match self.mode {
Mode::stringlit(n) => {
panic!("UNCLOSED STRING LITERAL STARTING ON LINE {}",n);
},
Mode::comment(n) => {
panic!("UNCLOSED COMMENT STARTING ON LINE {}",n);
},
_ => {return None;},
}//match self.mode
},
}//match nextline
} // read new line
let slice0 = &brline[self.column..];
let mut slice = slice0.trim_start();
let diff = slice0.len() - slice.len();
self.column += diff;
let (tok,ind) = self.next_token(slice);
let lcm = &self.line_comment[..]; // [..] makes slice borrow
match tok {
Symbol(s) if cmlen>0 && s.len()>=cmlen && &s[0..cmlen]==lcm => {
return_value = if self.keep_comments {Some(Verbatim(String::from(slice)))} else {Some(Nothing)};
self.column = brline.len();
},
Alphanum(a) if self.keywords.contains(&a) => {
return_value = Some(Keyword(a));
},
_ => { return_value = Some(tok); },
}
self.column += ind;
match return_value {
Some(Nothing) if !self.keep_nothing || self.current_string.len()>0 => {}, // keep looping
Some(NewLine) if !self.keep_newline => {},
_ => {break;}
}
}// do-while loop
return_value
}//next
}//impl Iterator for File_tokenizer
// currently cannot recognize string literals that span multiple lines...
// or multi-line comments (less important)
//////////////// integrate into RustLr:
/*
pub trait Token_translator<AT:Default>
{
fn translate_token(t:Token)->Lextoken<AT>;
}
impl<AT:Default> Lexer<AT> for File_tokenizer
{
fn nextsym(&mut self) -> Lextoken<AT>
{
Lextoken {
sym : String::from("num"),
value : AT::default(),
}
}
fn linenum(&self)-> usize { self.linenum }
}
*/