basic_lexer 0.1.2

#![allow(dead_code)]
#![allow(unused_variables)]
#![allow(non_snake_case)]
#![allow(non_camel_case_types)]
#![allow(unused_parens)]
#![allow(unused_mut)]
#![allow(unused_imports)]
use std::io::{Read,Error,BufRead,BufReader};
use std::fs::File;
use std::collections::{HashMap,HashSet};
// hand-build a lexer
use crate::Token::*;

#[derive(Clone,PartialEq,Debug)]
pub enum Token
{
   Integer(i64),
   Float(f64),
   Symbol(String),
   Alphanum(String),
   Keyword(String),
   Stringlit(String),
   Verbatim(String),
   Nothing,
}

fn isdelim(c:char) -> bool
{
  c=='(' || c=='[' || c=='{' || c==')' || c==']' || c=='}'
}


pub fn next_token(s:&str) -> (Token, usize) // returns token and next index
{
  //let st = s.trim_start();   // assume s already trimmed
//  if st.len()<0 {return (Nothing,0);}
  let first = s.chars().next().unwrap();
  let mut index = 0; //s.len() - st.len();
  if first.is_alphabetic() || first=='_' {
     index=match_alphanum(s);
     return (Alphanum(String::from(&s[0..index])),index);
  }//alphanumeric
  else if first=='.' || first.is_ascii_digit() { //possible number
     index = match_num(s);
     if index>0 {
        if let Ok(m) = s[0..index].parse::<i64>() {return (Integer(m),index);}
        else if let Ok(n) = s[0..index].parse::<f64>()
          {return (Float(n),index);}
     }
     else {return (Symbol(first.to_string()),1);}
  }
  else if first=='\"' {
     index = match_strlit(s);
     if index>0
       {return (Stringlit(String::from(&s[0..index])),index);}  //includes ""
     else {return (Verbatim(s.to_string()),s.len());}
  }
  else {
     index = match_symbol(s);
     if index>0 {return (Symbol(String::from(&s[0..index])),index);}
  }
  (Verbatim(s.to_string()),s.len())  // default
}

///// match token type, returns index of where token ended plus one.
pub fn match_alphanum(s:&str) -> usize
{
   if s.len()<1 {return 0;}
   let mut chars = s.chars();
   let mut next = chars.next().unwrap();
   if !(next.is_alphabetic() || next=='_') {return 0;}
   let mut index = 1;
   let mut stop=false;
   while !stop && index<s.len()
   {
      next = chars.next().unwrap();
      if next.is_alphanumeric() || next=='_' { index+=1; }
      else {stop=true;}
   }
   return index;
}//match_alphanum

fn match_base10(s:&str) -> usize  //not used
{
   let mut chars = s.chars();
   let mut index = 0;
   let mut stop = false;
   while !stop && index<s.len()
   {
     if chars.next().unwrap().is_ascii_digit() {index+=1}
     else {stop=true;}
   }
   return index;
}  // does not match +/- in front - returned as separate symbol

pub fn match_num(s:&str) -> usize  // could be integer or float
{
   let mut chars = s.chars();
   let mut index = 0;
   let mut stop = false;
   let mut foundpoint = false;
   let mut founddigit = false;
   while !stop && index<s.len()
   {
     let mut next = chars.next().unwrap();
     match (next, foundpoint) {
       ('.', false) => { foundpoint=true; index+=1},
       ('.', true) => {stop=true; index=0}, // match failed (2 .'s)
       (x,_) if x.is_ascii_digit() => { founddigit=true; index+=1},
       _ => {stop=true;}
     }//match
   }
   if founddigit {index} else {0}
}

pub fn match_strlit(s:&str) -> usize
{
   if s.len()<1 {return 0;}
   let mut chars = s.chars();
   let mut index = 0;
   let mut stop = false;
   let mut next = chars.next().unwrap();;
   if next !='\"' {return 0;} else { index+= 1; }
   while !stop && index<s.len()
   {
      next = chars.next().unwrap();
      index += 1;
      if next=='\\' && index<s.len() {
         next=chars.next().unwrap();
         index += 1;
      } //skip escape char
      else if next=='\"' { stop=true; }
      //else {index+=1; }
   }//while
   if stop {index} else {0}
}  // the stringlit will include the enclosing ""'s

pub fn match_symbol(s:&str) -> usize
{
   if s.len()<1 {return 0;}
   let mut chars = s.chars();
   let mut c = chars.next().unwrap();
   if isdelim(c) {return 1;}
   if c.is_ascii_digit() || c.is_alphanumeric() || c=='_' || c=='\"' || c=='.' {return 0;}
   let mut index = 1;
   let mut stop = false;
   while !stop && index<s.len()
   {
     c = chars.next().unwrap();
     if !c.is_whitespace() && !c.is_ascii_digit() && !c.is_alphanumeric() && c!='_' && c!='\"' && c!='.'
     {index+=1;} else {stop=true;}
   }
   return index;
}

/*
pub fn main()
{
  let input = "x=12 &&u2*20==.14 -> \"ab c \" + k; print(f); n.f();";
  println!("PARSING \"{}\"",input);
  let mut slice = input[0..].trim_start();
  while slice.len()>0
  {
     let (token,index) = next_token(slice);
     println!("token {:?}, ending before index {}",token,index);
     slice = slice[index..].trim_start();
  }
}//main
*/