Struct gluesql::core::sqlparser::tokenizer::Tokenizer

pub struct Tokenizer<'a> { /* private fields */ }

SQL Tokenizer

Implementations

impl<'a> Tokenizer<'a>

pub fn new( dialect: &'a (dyn Dialect + 'static), query: &'a str ) -> Tokenizer<'a>

Create a new SQL tokenizer for the specified SQL statement

let query = r#"SELECT 'foo'"#;

// Parsing the query
let tokens = Tokenizer::new(&dialect, &query).tokenize().unwrap();

assert_eq!(tokens, vec![
  Token::make_word("SELECT", None),
  Token::Whitespace(Whitespace::Space),
  Token::SingleQuotedString("foo".to_string()),
]);

pub fn with_unescape(self, unescape: bool) -> Tokenizer<'a>

Set unescape mode

When true (default) the tokenizer unescapes literal values (for example, "" in SQL is unescaped to the literal ").

When false, the tokenizer provides the raw strings as provided in the query. This can be helpful for programs that wish to recover the exact original query text without normalizing the escaping

Example

let query = r#""Foo "" Bar""#;
let unescaped = Token::make_word(r#"Foo " Bar"#, Some('"'));
let original  = Token::make_word(r#"Foo "" Bar"#, Some('"'));

// Parsing with unescaping (default)
let tokens = Tokenizer::new(&dialect, &query).tokenize().unwrap();
assert_eq!(tokens, vec![unescaped]);

// Parsing with unescape = false
let tokens = Tokenizer::new(&dialect, &query)
   .with_unescape(false)
   .tokenize().unwrap();
assert_eq!(tokens, vec![original]);

pub fn tokenize(&mut self) -> Result<Vec<Token>, TokenizerError>

Tokenize the statement and produce a vector of tokens

pub fn tokenize_with_location( &mut self ) -> Result<Vec<TokenWithLocation>, TokenizerError>

Tokenize the statement and produce a vector of tokens with location information