Function tokenize 

pub fn tokenize(code: &str) -> Vec<Token>

Split code into tokens covering every byte, in order.

Never fails: an unterminated string, comment, quoted name or Jinja block runs to the end of the input.

use rich_ext::sql::{tokenize, TokenKind};

let code = "select \"a b\" from t -- note";
let kinds: Vec<(TokenKind, &str)> = tokenize(code)
    .into_iter()
    .filter(|t| t.kind != TokenKind::Whitespace)
    .map(|t| (t.kind, &code[t.range]))
    .collect();
assert_eq!(kinds, [
    (TokenKind::Keyword, "select"),
    (TokenKind::QuotedName, "\"a b\""),
    (TokenKind::Keyword, "from"),
    (TokenKind::Name, "t"),
    (TokenKind::Comment, "-- note"),
]);