use chumsky::prelude::*;
#[derive(Debug, Clone, PartialEq, Eq)]
pub(super) enum Token {
Word(String),
QuotedIdent(String),
Literal(String),
Number(String),
Star,
LParen,
RParen,
Other,
}
pub(super) fn lex(sql: &str) -> Vec<Token> {
sql_tokens().parse(sql).into_result().unwrap_or_default()
}
type E<'src> = extra::Err<Rich<'src, char>>;
fn sql_tokens<'src>() -> impl Parser<'src, &'src str, Vec<Token>, E<'src>> {
let number = any()
.filter(|c: &char| c.is_ascii_digit())
.repeated()
.at_least(1)
.to_slice()
.map(|s: &str| Token::Number(s.to_owned()));
let token = choice((
word(),
double_quoted_ident(),
number,
just('*').to(Token::Star),
just('(').to(Token::LParen),
just(')').to(Token::RParen),
single_quoted_string().map(Token::Literal),
dollar_quoted_string().to(Token::Other),
any().to(Token::Other),
));
noise().ignore_then(token.then_ignore(noise()).repeated().collect())
}
fn word<'src>() -> impl Parser<'src, &'src str, Token, E<'src>> + Clone {
any()
.filter(|c: &char| c.is_ascii_alphabetic() || *c == '_')
.then(
any()
.filter(|c: &char| c.is_ascii_alphanumeric() || *c == '_')
.repeated(),
)
.to_slice()
.map(|s: &str| Token::Word(s.to_ascii_lowercase()))
}
fn double_quoted_ident<'src>() -> impl Parser<'src, &'src str, Token, E<'src>> + Clone {
just('"')
.ignore_then(custom(|inp| {
let mut name = String::new();
loop {
match inp.next() {
Some('"') if inp.peek() == Some('"') => {
inp.skip();
name.push('"'); }
Some('"') | None => return Ok(name), Some(c) => name.push(c),
}
}
}))
.map(Token::QuotedIdent)
}
fn noise<'src>() -> impl Parser<'src, &'src str, (), E<'src>> + Clone {
choice((
any()
.filter(|c: &char| c.is_ascii_whitespace())
.repeated()
.at_least(1)
.to(()),
line_comment(),
block_comment(),
))
.repeated()
.to(())
}
fn line_comment<'src>() -> impl Parser<'src, &'src str, (), E<'src>> + Clone {
just("--")
.then(any().filter(|c: &char| *c != '\n').repeated())
.to(())
}
fn block_comment<'src>() -> impl Parser<'src, &'src str, (), E<'src>> + Clone {
just("/*")
.ignore_then(custom(|inp| {
let mut depth = 1u32;
while depth > 0 {
match inp.next() {
Some('/') => {
if inp.peek() == Some('*') {
inp.skip();
depth += 1;
}
}
Some('*') => {
if inp.peek() == Some('/') {
inp.skip();
depth -= 1;
}
}
Some(_) => {}
None => break, }
}
Ok(())
}))
.to(())
}
fn single_quoted_string<'src>() -> impl Parser<'src, &'src str, String, E<'src>> + Clone {
just('\'').ignore_then(custom(|inp| {
let mut content = String::new();
loop {
match inp.next() {
Some('\'') if inp.peek() == Some('\'') => {
inp.skip();
content.push('\'');
}
Some('\'') | None => return Ok(content.to_ascii_lowercase()),
Some(c) => content.push(c),
}
}
}))
}
fn dollar_quoted_string<'src>() -> impl Parser<'src, &'src str, (), E<'src>> + Clone {
just('$').ignore_then(custom(|inp| {
let mut tag = String::from("$");
loop {
match inp.next() {
Some('$') => {
tag.push('$');
break;
}
Some(c) if c.is_ascii_alphanumeric() || c == '_' => {
tag.push(c);
}
_ => return Ok(()),
}
}
let tag_bytes = tag.as_bytes();
let mut matched = 0usize;
loop {
let next: Option<char> = inp.next();
let Some(c) = next else {
return Ok(()); };
if !c.is_ascii() {
matched = 0;
continue;
}
let b = c as u8;
if matched > 0 && b != tag_bytes[matched] {
matched = 0;
}
if b == tag_bytes[matched] {
matched += 1;
if matched == tag_bytes.len() {
return Ok(());
}
}
}
}))
}
#[cfg(test)]
mod tests {
use super::*;
fn t(sql: &str) -> Vec<Token> {
lex(sql)
}
#[test]
fn words_are_lowercased() {
assert_eq!(t("SELECT"), vec![Token::Word("select".into())]);
assert_eq!(t("Begin"), vec![Token::Word("begin".into())]);
}
#[test]
fn quoted_idents_preserve_case() {
assert_eq!(
t(r#""MyTable""#),
vec![Token::QuotedIdent("MyTable".into())]
);
assert_eq!(
t(r#""has""quote""#),
vec![Token::QuotedIdent(r#"has"quote"#.into())]
);
}
#[test]
fn single_quoted_strings_are_lowercased() {
assert_eq!(t("'Off'"), vec![Token::Literal("off".into())]);
assert_eq!(t("'it''s ok'"), vec![Token::Literal("it's ok".into())]);
}
#[test]
fn nested_block_comments_are_skipped() {
assert_eq!(
t("/* outer /* inner */ still */ SELECT"),
vec![Token::Word("select".into())]
);
}
#[test]
fn dollar_quoted_strings_are_skipped() {
assert_eq!(
t("$body$SELECT 1$body$ WORD"),
vec![Token::Other, Token::Word("word".into())]
);
}
#[test]
fn dollar_quoted_body_with_unicode_low_byte_collision() {
assert_eq!(
t("$a$Ĥš$XYZ$a$ WORD"),
vec![Token::Other, Token::Word("word".into())]
);
}
#[test]
fn dollar_quoted_tag_with_nested_dollar_prefix() {
assert_eq!(
t("$ab$$$ab$ WORD"),
vec![Token::Other, Token::Word("word".into())]
);
}
#[test]
fn unterminated_inputs_do_not_panic() {
let _ = t("/* unterminated");
let _ = t("'unterminated");
let _ = t("$$unterminated");
let _ = t(r#""unterminated"#);
}
}