powr-tokenizer 0.3.0

converts JS/TS code into tokens. part of the power project.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25

use std::{env, process};

use powr_tokenizer::Tokenizer;

fn main() {
    let args: Vec<String> = env::args().collect();

    if args.len() <= 1 {
        eprintln!("please provide a string of characters!");
        process::exit(1);
    }

    (1..args.len()).into_iter().for_each(|i| {
        let string = args.get(i).unwrap().to_owned();
        let chars: Vec<char> = string.chars().collect();

        let tokenizer = Tokenizer::new(chars);

        for token in tokenizer {
            println!("{:#?}", token);
        }

        println!();
    });
}