perl-lexer 0.13.2

High-performance Perl lexer with context-aware tokenization
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13

//! Debug lexer output
use perl_lexer::PerlLexer;

fn main() {
    let code = "package Test::Module;";
    println!("Code: {}", code);
    println!("\nLexer tokens:");

    let mut lexer = PerlLexer::new(code);
    while let Some(token) = lexer.next_token() {
        println!("  Token: {:?} '{}'", token.token_type, token.text);
    }
}