Struct Tokenizer 

pub struct Tokenizer {}

SQL lexer

Implementations

impl Tokenizer

pub fn new() -> Self

Constructor

Trait Implementations

impl Default for Tokenizer

fn default() -> Tokenizer

Returns the “default value” for a type.

impl Splitter for Tokenizer

use sqlite3_parser::lexer::sql::Tokenizer;
use sqlite3_parser::lexer::Scanner;

let tokenizer = Tokenizer::new();
let input = b"PRAGMA parser_trace=ON;";
let mut s = Scanner::new(tokenizer);
let Ok((_, Some((token1, _)), _)) = s.scan(input) else { panic!() };
s.scan(input).unwrap();
assert!(b"PRAGMA".eq_ignore_ascii_case(token1));

type Error = Error

Potential error raised

type TokenType = TokenType

Token generated

fn split<'input>( &mut self, data: &'input [u8], ) -> Result<(Option<Token<'input>>, usize), Error>

The arguments are an initial substring of the remaining unprocessed data.