pub struct Tokenizer<'a> { /* private fields */ }
Expand description
SQL Tokenizer
Implementations§
source§impl<'a> Tokenizer<'a>
impl<'a> Tokenizer<'a>
sourcepub fn new(dialect: &'a dyn Dialect, query: &'a str) -> Self
pub fn new(dialect: &'a dyn Dialect, query: &'a str) -> Self
Create a new SQL tokenizer for the specified SQL statement
let query = r#"SELECT 'foo'"#;
// Parsing the query
let tokens = Tokenizer::new(&dialect, &query).tokenize().unwrap();
assert_eq!(tokens, vec![
Token::make_word("SELECT", None),
Token::Whitespace(Whitespace::Space),
Token::SingleQuotedString("foo".to_string()),
]);
sourcepub fn with_unescape(self, unescape: bool) -> Self
pub fn with_unescape(self, unescape: bool) -> Self
Set unescape mode
When true (default) the tokenizer unescapes literal values
(for example, ""
in SQL is unescaped to the literal "
).
When false, the tokenizer provides the raw strings as provided in the query. This can be helpful for programs that wish to recover the exact original query text without normalizing the escaping
Example
let query = r#""Foo "" Bar""#;
let unescaped = Token::make_word(r#"Foo " Bar"#, Some('"'));
let original = Token::make_word(r#"Foo "" Bar"#, Some('"'));
// Parsing with unescaping (default)
let tokens = Tokenizer::new(&dialect, &query).tokenize().unwrap();
assert_eq!(tokens, vec![unescaped]);
// Parsing with unescape = false
let tokens = Tokenizer::new(&dialect, &query)
.with_unescape(false)
.tokenize().unwrap();
assert_eq!(tokens, vec![original]);
sourcepub fn tokenize(&mut self) -> Result<Vec<Token>, TokenizerError>
pub fn tokenize(&mut self) -> Result<Vec<Token>, TokenizerError>
Tokenize the statement and produce a vector of tokens
sourcepub fn tokenize_with_location(
&mut self
) -> Result<Vec<TokenWithLocation>, TokenizerError>
pub fn tokenize_with_location( &mut self ) -> Result<Vec<TokenWithLocation>, TokenizerError>
Tokenize the statement and produce a vector of tokens with location information
Auto Trait Implementations§
impl<'a> !RefUnwindSafe for Tokenizer<'a>
impl<'a> !Send for Tokenizer<'a>
impl<'a> !Sync for Tokenizer<'a>
impl<'a> Unpin for Tokenizer<'a>
impl<'a> !UnwindSafe for Tokenizer<'a>
Blanket Implementations§
source§impl<T> BorrowMut<T> for Twhere
T: ?Sized,
impl<T> BorrowMut<T> for Twhere
T: ?Sized,
source§fn borrow_mut(&mut self) -> &mut T
fn borrow_mut(&mut self) -> &mut T
Mutably borrows from an owned value. Read more