gitql_parser::tokenizer

Function tokenize

Source
pub fn tokenize(script: String) -> Result<Vec<Token>, Box<Diagnostic>>