basic_lexer 0.1.2

Basic lexical analyzer for use in parsing and compiling
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107

extern crate regex;
use regex::Regex;

///////////////////////////////////////////////

pub struct Tokenizer
{
  int_re : Regex,  // to be generated from re_str
  float_re : Regex,
  sym_re : Regex,
  alpha_re : Regex,
  kw_re    : Regex,
  str_re   : Regex,
  pub keywords: String,
  pub symbols: String,
}
impl Tokenizer
{

//   fn new() -> Tokenizer
//   {  Tokenizer::default() }

   fn addkeyword(&mut self, kw:&str)
   {  self.keywords.push('|');
      self.keywords.push_str(kw);
      self.kw_re = Regex::new(&self.keywords).unwrap();
   }
   fn addsymbol(&mut self, sym:&str)
   {
    if sym.len()>1 {
      self.symbols.push('|');
      self.symbols.push_str(sym);
      self.sym_re = Regex::new(&self.symbols).unwrap();
     }
   }
}//impl tokenizer

impl Default for Tokenizer
{
  fn default() -> Self
  {
     let kws = "if|while|else|let|lambda|for";
     let syms = "==|<=|>=|!=|::|&&|(||)|-->|[+-*/%&|@#$^~.,;]";
     Tokenizer {
        kw_re : Regex::new(kws).unwrap(),
        sym_re : Regex::new(syms).unwrap(),
        int_re : Regex::new(r"[+-]??\d+").unwrap(),
        float_re: Regex::new(r"[+-]??\d*\.\d*").unwrap(),
        alpha_re: Regex::new(r"[[:alpha:]][[:alnum:]]*").unwrap(),
        str_re: Regex::new("\"[^\"]\"").unwrap(),
        keywords: kws.to_owned(),
        symbols : syms.to_owned(),
     }
  }
}//impl Default

// split by reg. expression, keep splits if not whitespace
pub fn splitby<'t>(s:&'t str, re:&Regex) 
{
   let split:Vec<_> = re.split(s).collect();
   //let matches = s.match_indices(re);
}


pub fn main2()
{
  let input ="abc=123-4*5/1.6 ok [--]";
  let split_re = Regex::new(r"\*|\s|[-/=.]").unwrap();
  let split = split_re.split(input).into_iter();
  for x in split { println!("({})",x); }
}//main



/*
fn testing()
{
  let reader = match File::open("Cargo.toml") {
          Ok(fi) => BufReader::new(fi),
          _ => {panic!("No such file ");},
        };
//  let r2 = reader;
  let ls = reader.lines(); // induce compiler error showing type
  let ls2 = ls;
  let ls3 = ls; // induce compiler error showing type
}
*/


/*  must implement dependent traits this way:
trait T1 {
  fn f()->i32;
}
trait T2 : T1  // T2 can be implemented only if T1 is implemented
{
  fn g();
}
impl<'t> T2 for Str_tokenizer<'t>
{
   fn g() {}
   //fn f()->i32 {1} //can't just put this here
}
impl<'t> T1 for Str_tokenizer<'t>
{
   fn f()->i32 {1}
}
*/