sentience-tokenize 0.2.3

Tiny Rust zero-dep tokenizer (ident, number, string, parens, operators, keywords).
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31

use sentience_tokenize::{tokenize_iter, TokenKind};
use std::io::{self, Read};

fn main() {
    let mut src = String::new();
    if io::stdin().read_to_string(&mut src).is_err() {
        eprintln!("failed to read stdin");
        std::process::exit(1);
    }

    for item in tokenize_iter(&src) {
        match item {
            Ok(tok) => match tok.kind {
                TokenKind::Ident(ref s) => {
                    println!("Ident(\"{}\") @{}..{}", s, tok.span.start, tok.span.end)
                }
                TokenKind::Number(ref s) => {
                    println!("Number(\"{}\") @{}..{}", s, tok.span.start, tok.span.end)
                }
                TokenKind::String(ref s) => {
                    println!("String(\"{}\") @{}..{}", s, tok.span.start, tok.span.end)
                }
                ref other => println!("{:?} @{}..{}", other, tok.span.start, tok.span.end),
            },
            Err(e) => {
                eprintln!("error: {}", e);
                std::process::exit(2);
            }
        }
    }
}