Struct Tokenizer

pub struct Tokenizer {}

Implementations

impl Tokenizer

pub fn new() -> Tokenizer

Trait Implementations

impl Default for Tokenizer

fn default() -> Tokenizer

Returns the “default value” for a type.

impl Splitter for Tokenizer

use sqlite3_parser::lexer::sql::Tokenizer;
use sqlite3_parser::lexer::Scanner;

let tokenizer = Tokenizer::new();
let input = "PRAGMA parser_trace=ON;".as_bytes();
let mut s = Scanner::new(input, tokenizer);
let (token1, _) = s.scan().unwrap().unwrap();
s.scan().unwrap().unwrap();
assert!(b"PRAGMA".eq_ignore_ascii_case(token1));

type Error = Error

type TokenType = TokenType

fn split<'input>( &mut self, data: &'input [u8], ) -> Result<(Option<Token<'input>>, usize), Error>

The arguments are an initial substring of the remaining unprocessed data.