Struct deltalake::datafusion::sql::sqlparser::tokenizer::Tokenizer

pub struct Tokenizer<'a> { /* private fields */ }

SQL Tokenizer

Implementations

impl<'a> Tokenizer<'a>

pub fn new( dialect: &'a (dyn Dialect + 'static), query: &'a str ) -> Tokenizer<'a>

Create a new SQL tokenizer for the specified SQL statement

let query = r#"SELECT 'foo'"#;

// Parsing the query
let tokens = Tokenizer::new(&dialect, &query).tokenize().unwrap();

assert_eq!(tokens, vec![
  Token::make_word("SELECT", None),
  Token::Whitespace(Whitespace::Space),
  Token::SingleQuotedString("foo".to_string()),
]);

pub fn with_unescape(self, unescape: bool) -> Tokenizer<'a>

Set unescape mode

When true (default) the tokenizer unescapes literal values (for example, "" in SQL is unescaped to the literal ").

When false, the tokenizer provides the raw strings as provided in the query. This can be helpful for programs that wish to recover the exact original query text without normalizing the escaping

Example

let query = r#""Foo "" Bar""#;
let unescaped = Token::make_word(r#"Foo " Bar"#, Some('"'));
let original  = Token::make_word(r#"Foo "" Bar"#, Some('"'));

// Parsing with unescaping (default)
let tokens = Tokenizer::new(&dialect, &query).tokenize().unwrap();
assert_eq!(tokens, vec![unescaped]);

// Parsing with unescape = false
let tokens = Tokenizer::new(&dialect, &query)
   .with_unescape(false)
   .tokenize().unwrap();
assert_eq!(tokens, vec![original]);

pub fn tokenize(&mut self) -> Result<Vec<Token>, TokenizerError>

Tokenize the statement and produce a vector of tokens

pub fn tokenize_with_location( &mut self ) -> Result<Vec<TokenWithLocation>, TokenizerError>

Tokenize the statement and produce a vector of tokens with location information

pub fn tokenize_with_location_into_buf( &mut self, buf: &mut Vec<TokenWithLocation> ) -> Result<(), TokenizerError>

Tokenize the statement and append tokens with location information into the provided buffer. If an error is thrown, the buffer will contain all tokens that were successfully parsed before the error.