Function rustpython_parser::lexer::lex

pub fn lex(source: &str, mode: Mode) -> impl Iterator<Item = LexResult> + '_

Create a new lexer from a source string.

Examples

use rustpython_parser::{Mode, lexer::lex};

let source = "def hello(): return 'world'";
let lexer = lex(source, Mode::Module);

for token in lexer {
   println!("{:?}", token);
}