pub struct Tokenizer<'a> { /* private fields */ }
Expand description
SQL Tokenizer
Implementations§
source§impl<'a> Tokenizer<'a>
impl<'a> Tokenizer<'a>
sourcepub fn new(dialect: &'a dyn Dialect, query: &'a str) -> Self
pub fn new(dialect: &'a dyn Dialect, query: &'a str) -> Self
Create a new SQL tokenizer for the specified SQL statement
let query = r#"SELECT 'foo'"#;
// Parsing the query
let tokens = Tokenizer::new(&dialect, &query).tokenize().unwrap();
assert_eq!(tokens, vec![
Token::make_word("SELECT", None),
Token::Whitespace(Whitespace::Space),
Token::SingleQuotedString("foo".to_string()),
]);
sourcepub fn with_unescape(self, unescape: bool) -> Self
pub fn with_unescape(self, unescape: bool) -> Self
Set unescape mode
When true (default) the tokenizer unescapes literal values
(for example, ""
in SQL is unescaped to the literal "
).
When false, the tokenizer provides the raw strings as provided in the query. This can be helpful for programs that wish to recover the exact original query text without normalizing the escaping
§Example
let query = r#""Foo "" Bar""#;
let unescaped = Token::make_word(r#"Foo " Bar"#, Some('"'));
let original = Token::make_word(r#"Foo "" Bar"#, Some('"'));
// Parsing with unescaping (default)
let tokens = Tokenizer::new(&dialect, &query).tokenize().unwrap();
assert_eq!(tokens, vec![unescaped]);
// Parsing with unescape = false
let tokens = Tokenizer::new(&dialect, &query)
.with_unescape(false)
.tokenize().unwrap();
assert_eq!(tokens, vec![original]);
sourcepub fn tokenize(&mut self) -> Result<Vec<Token>, TokenizerError>
pub fn tokenize(&mut self) -> Result<Vec<Token>, TokenizerError>
Tokenize the statement and produce a vector of tokens
sourcepub fn tokenize_with_location(
&mut self
) -> Result<Vec<TokenWithLocation>, TokenizerError>
pub fn tokenize_with_location( &mut self ) -> Result<Vec<TokenWithLocation>, TokenizerError>
Tokenize the statement and produce a vector of tokens with location information
sourcepub fn tokenize_with_location_into_buf(
&mut self,
buf: &mut Vec<TokenWithLocation>
) -> Result<(), TokenizerError>
pub fn tokenize_with_location_into_buf( &mut self, buf: &mut Vec<TokenWithLocation> ) -> Result<(), TokenizerError>
Tokenize the statement and append tokens with location information into the provided buffer. If an error is thrown, the buffer will contain all tokens that were successfully parsed before the error.
Auto Trait Implementations§
impl<'a> Freeze for Tokenizer<'a>
impl<'a> !RefUnwindSafe for Tokenizer<'a>
impl<'a> !Send for Tokenizer<'a>
impl<'a> !Sync for Tokenizer<'a>
impl<'a> Unpin for Tokenizer<'a>
impl<'a> !UnwindSafe for Tokenizer<'a>
Blanket Implementations§
source§impl<T> BorrowMut<T> for Twhere
T: ?Sized,
impl<T> BorrowMut<T> for Twhere
T: ?Sized,
source§fn borrow_mut(&mut self) -> &mut T
fn borrow_mut(&mut self) -> &mut T
Mutably borrows from an owned value. Read more