perl-lexer 0.13.2

High-performance Perl lexer with context-aware tokenization
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16

use perl_lexer::{PerlLexer, TokenType};

fn main() {
    let input = "print \"hello\\n\";\n__DATA__\nnot perl code";
    let mut lexer = PerlLexer::new(input);

    println!("Tokenizing: {:?}", input);
    println!();

    while let Some(token) = lexer.next_token() {
        println!("Token: {:?} = {:?}", token.token_type, &input[token.start..token.end]);
        if matches!(token.token_type, TokenType::EOF) {
            break;
        }
    }
}