This crate provides Tokenizer, which iterates over tokens in a SQL string as SQLite would.

Example

extern crate sqlite3_tokenizer;

use sqlite3_tokenizer::Tokenizer;

fn main() {
    for token in Tokenizer::new("SELECT * FROM t") {
        println!("Token of kind {:?} is written {:?}", token.kind, token.text);
    }
}

outputs

Token of kind Select is written "SELECT"
Token of kind Space is written " "
Token of kind Star is written "*"
Token of kind Space is written " "
Token of kind From is written "FROM"
Token of kind Space is written " "
Token of kind Id is written "t"