Function lex

pub fn lex(
    source: &str,
    mode: Mode,
) -> impl Iterator<Item = Result<(Tok, TextRange), LexicalError>>

Create a new lexer from a source string.

Examples

use rustpython_parser::{Mode, lexer::lex};

let source = "def hello(): return 'world'";
let lexer = lex(source, Mode::Module);

for token in lexer {
   println!("{:?}", token);
}