use crate::ast::Span;
use crate::ast::dialect::{
FeatureSet,
lex_class::{
CLASS_DIGIT, CLASS_IDENTIFIER_CONTINUE, CLASS_IDENTIFIER_START, CLASS_OPERATOR,
CLASS_PUNCTUATION, CLASS_WHITESPACE, CLASS_WHITESPACE_BOUNDARY, CLASS_WHITESPACE_CONTINUE,
},
lookup_keyword,
};
use super::cursor::Cursor;
use super::error::{LexError, LexErrorKind};
use super::token::{Operator, Punctuation, Token, TokenKind};
use super::trivia::{TriviaKind, TriviaRange, TriviaSink};
#[derive(Clone, Copy, Debug, Default)]
pub(crate) struct LexState {
versioned_region_start: Option<u32>,
}
pub(crate) fn next_token<S: TriviaSink>(
cursor: &mut Cursor,
features: &FeatureSet,
trivia: &mut S,
state: &mut LexState,
) -> Result<Option<Token>, LexError> {
skip_trivia(cursor, features, trivia, state)?;
let start = cursor.pos();
let Some(byte) = cursor.peek() else {
if let Some(open) = state.versioned_region_start {
return Err(LexError::new(
LexErrorKind::UnterminatedBlockComment,
Span::new(open, cursor.pos()),
));
}
return Ok(None);
};
let token = match byte {
b'E' | b'e'
if features.string_literals.escape_strings && cursor.peek_nth(1) == Some(b'\'') =>
{
let (token, continued) = scan_prefixed_string(cursor, 1, b'\'', true)?;
let text = cursor
.src()
.get(token.span.start() as usize..token.span.end() as usize);
let invalid = text.is_some_and(|text| {
if continued {
!escape_string_segments_are_valid(text)
} else {
!crate::ast::postgres_escape_string_is_valid(text)
}
});
if invalid {
return Err(LexError::new(
LexErrorKind::InvalidEscapeSequence,
token.span,
));
}
token
}
b'N' | b'n'
if features.string_literals.national_strings && cursor.peek_nth(1) == Some(b'\'') =>
{
scan_prefixed_string(cursor, 1, b'\'', features.string_literals.backslash_escapes)?.0
}
b'U' | b'u'
if features.string_literals.unicode_strings
&& cursor.peek_nth(1) == Some(b'&')
&& cursor.peek_nth(2) == Some(b'\'') =>
{
scan_prefixed_string(cursor, 2, b'\'', false)?.0
}
b'U' | b'u'
if features.string_literals.unicode_strings
&& cursor.peek_nth(1) == Some(b'&')
&& cursor.peek_nth(2) == Some(b'"') =>
{
scan_unicode_ident(cursor, features)?
}
b'X' | b'x'
if features.string_literals.blob_literals && cursor.peek_nth(1) == Some(b'\'') =>
{
scan_hex_blob(cursor)?
}
b'B' | b'b' | b'X' | b'x'
if features.string_literals.bit_string_literals
&& cursor.peek_nth(1) == Some(b'\'') =>
{
scan_prefixed_string(cursor, 1, b'\'', false)?.0
}
b'_' if features.string_literals.charset_introducers => {
match charset_introducer_prefix(cursor, features) {
Some((prefix_len, close)) => {
scan_prefixed_string(
cursor,
prefix_len,
close,
features.string_literals.backslash_escapes,
)?
.0
}
None => scan_word(cursor, features),
}
}
b'\'' => scan_quoted(
cursor,
QuoteScan {
close: b'\'',
kind: TokenKind::String,
unterminated: LexErrorKind::UnterminatedString,
backslash: features.string_literals.backslash_escapes,
allow_zero_length: false,
},
)?,
b'"' if features.string_literals.double_quoted_strings => scan_quoted(
cursor,
QuoteScan {
close: b'"',
kind: TokenKind::String,
unterminated: LexErrorKind::UnterminatedString,
backslash: features.string_literals.backslash_escapes,
allow_zero_length: false,
},
)?,
byte if opening_identifier_quote(features, byte).is_some() => {
let close = opening_identifier_quote(features, byte)
.expect("guard matched an opening identifier quote");
scan_quoted(
cursor,
QuoteScan {
close,
kind: TokenKind::QuotedIdent,
unterminated: LexErrorKind::UnterminatedQuotedIdent,
backslash: false,
allow_zero_length: features.identifier_syntax.empty_quoted_identifiers,
},
)?
}
b'$' if features.numeric_literals.money_literals && money_follows(cursor, features) => {
scan_money(cursor, features)
}
b'$' if features.parameters.positional_dollar
&& cursor.peek_nth(1).is_some_and(|b| b.is_ascii_digit()) =>
{
scan_parameter(cursor, features)
}
b'$' if features.parameters.named_dollar
&& cursor
.char_at(1)
.is_some_and(|ch| is_identifier_start(ch, features)) =>
{
scan_parameter(cursor, features)
}
b'$' if features.string_literals.dollar_quoted_strings => {
scan_dollar_quote(cursor, features)?
}
b'$' => {
return Err(LexError::new(
LexErrorKind::StrayByte,
Span::new(start, start + 1),
));
}
b'?' if features.parameters.anonymous_question => scan_parameter(cursor, features),
b'?' if features.parameters.numbered_question
&& cursor.peek_nth(1).is_some_and(|b| b.is_ascii_digit()) =>
{
scan_parameter(cursor, features)
}
b'?' if features.operator_syntax.jsonb_operators => scan_operator(cursor, features),
b':' if features.parameters.named_colon
&& cursor
.char_at(1)
.is_some_and(|ch| is_identifier_start(ch, features)) =>
{
scan_parameter(cursor, features)
}
b'@' if features.utility_syntax.stage_references
&& (matches!(cursor.peek_nth(1), Some(b'~') | Some(b'%'))
|| cursor
.char_at(1)
.is_some_and(|ch| is_identifier_start(ch, features))) =>
{
scan_stage_reference(cursor, features)
}
b'@' if features.session_variables.system_variables
&& cursor.peek_nth(1) == Some(b'@')
&& cursor
.char_at(2)
.is_some_and(|ch| is_identifier_start(ch, features)) =>
{
scan_session_variable(cursor, features)
}
b'@' if features.session_variables.user_variables
&& cursor
.char_at(1)
.is_some_and(|ch| is_identifier_start(ch, features)) =>
{
scan_session_variable(cursor, features)
}
b'@' if features.parameters.named_at
&& cursor
.char_at(1)
.is_some_and(|ch| is_identifier_start(ch, features)) =>
{
scan_parameter(cursor, features)
}
b'@' if features.operator_syntax.containment_operators
&& cursor.peek_nth(1) == Some(b'>') =>
{
scan_operator(cursor, features)
}
b'@' if features.operator_syntax.jsonb_operators
&& matches!(cursor.peek_nth(1), Some(b'?') | Some(b'@')) =>
{
scan_operator(cursor, features)
}
b'@' if features.operator_syntax.custom_operators => scan_operator(cursor, features),
b'@' if features.session_variables.user_variables => {
let start = cursor.pos();
cursor.bump(); Token::new(
TokenKind::Punctuation(Punctuation::At),
Span::new(start, cursor.pos()),
)
}
_ if features.has_byte_class(byte, CLASS_IDENTIFIER_START) => scan_word(cursor, features),
_ if byte >= 0x80 => {
let ch = cursor
.char_at(0)
.expect("a non-ASCII lead byte begins a valid UTF-8 char in &str source");
if is_identifier_start(ch, features) {
scan_word(cursor, features)
} else {
return Err(LexError::new(
LexErrorKind::StrayByte,
Span::new(start, start + ch.len_utf8() as u32),
));
}
}
_ if is_digit(byte, features) => scan_number(cursor, features)?,
b'.' if cursor.peek_nth(1).is_some_and(|b| is_digit(b, features)) => {
scan_number(cursor, features)?
}
b'#' if features.expression_syntax.positional_column
&& cursor.peek_nth(1).is_some_and(|b| b.is_ascii_digit()) =>
{
scan_positional_column(cursor)
}
b'#' if features.hash_bitwise_xor => scan_operator(cursor, features),
b'`' if features.operator_syntax.custom_operators => scan_operator(cursor, features),
_ if features.has_byte_class(byte, CLASS_OPERATOR) => scan_operator(cursor, features),
_ if features.has_byte_class(byte, CLASS_PUNCTUATION) => scan_punctuation(cursor, features),
_ => {
return Err(LexError::new(
LexErrorKind::StrayByte,
Span::new(start, start + 1),
));
}
};
let nul_byte_error = match token.kind {
TokenKind::String => Some(LexErrorKind::NulByteInString),
TokenKind::QuotedIdent => Some(LexErrorKind::NulByteInIdentifier),
_ => None,
};
if let Some(error_kind) = nul_byte_error {
let text = &cursor.src()[token.span.start() as usize..token.span.end() as usize];
if crate::ast::string_literal_embeds_nul(text) {
return Err(LexError::new(error_kind, token.span));
}
}
Ok(Some(token))
}
fn skip_trivia<S: TriviaSink>(
cursor: &mut Cursor,
features: &FeatureSet,
trivia: &mut S,
state: &mut LexState,
) -> Result<(), LexError> {
let track_boundary = features.byte_classes.has_boundary_whitespace();
let sweep_start = cursor.pos();
let content_before_sweep =
sweep_start != 0 && cursor.src().as_bytes()[sweep_start as usize - 1] != b';';
let mut comment_seen = false;
let mut pending_boundary: Option<u32> = None;
loop {
let start = cursor.pos();
let kind = match cursor.peek() {
Some(byte) if features.has_byte_class(byte, CLASS_WHITESPACE) => {
cursor.eat_while(|b| {
features.has_byte_class(b, CLASS_WHITESPACE | CLASS_WHITESPACE_CONTINUE)
});
if track_boundary {
let boundary_at = cursor.src().as_bytes()
[start as usize..cursor.pos() as usize]
.iter()
.position(|&b| features.has_byte_class(b, CLASS_WHITESPACE_BOUNDARY))
.map(|i| start + i as u32);
if pending_boundary.is_none() && (content_before_sweep || comment_seen) {
pending_boundary = boundary_at;
}
}
TriviaKind::Whitespace
}
Some(b'-') if cursor.peek_nth(1) == Some(b'-') => {
skip_line_comment(
cursor,
2,
features.comment_syntax.line_comment_ends_at_carriage_return,
);
TriviaKind::LineComment
}
Some(b'#') if features.comment_syntax.line_comment_hash => {
skip_line_comment(
cursor,
1,
features.comment_syntax.line_comment_ends_at_carriage_return,
);
TriviaKind::LineComment
}
Some(b'*')
if state.versioned_region_start.is_some() && cursor.peek_nth(1) == Some(b'/') =>
{
cursor.advance_bytes(2);
state.versioned_region_start = None;
TriviaKind::BlockComment
}
Some(b'/')
if cursor.peek_nth(1) == Some(b'*')
&& !(features.comment_syntax.unterminated_block_comment_at_eof
&& cursor.peek_nth(2).is_none()) =>
{
match features.comment_syntax.versioned_comments {
Some(bound) if cursor.peek_nth(2) == Some(b'!') => {
scan_versioned_marker(
cursor,
bound,
features.comment_syntax.nested_block_comments,
state,
)?;
}
_ => {
skip_block_comment(
cursor,
features.comment_syntax.nested_block_comments,
features.comment_syntax.unterminated_block_comment_at_eof,
)?;
}
}
TriviaKind::BlockComment
}
_ => {
if let Some(off) = pending_boundary {
let content_after = cursor.peek().is_some_and(|b| b != b';');
if content_after {
return Err(LexError::new(
LexErrorKind::StrayByte,
Span::new(off, off + 1),
));
}
}
return Ok(());
}
};
if matches!(kind, TriviaKind::LineComment | TriviaKind::BlockComment) {
if let Some(off) = pending_boundary {
return Err(LexError::new(
LexErrorKind::StrayByte,
Span::new(off, off + 1),
));
}
comment_seen = true;
}
if matches!(kind, TriviaKind::LineComment | TriviaKind::BlockComment)
&& cursor.src().as_bytes()[start as usize..cursor.pos() as usize].contains(&0)
{
return Err(LexError::new(
LexErrorKind::NulByteInComment,
Span::new(start, cursor.pos()),
));
}
if S::RECORDING {
trivia.record(TriviaRange::new(kind, Span::new(start, cursor.pos())));
}
}
}
fn skip_line_comment(cursor: &mut Cursor, marker_len: u32, cr_terminates: bool) {
cursor.advance_bytes(marker_len); if cr_terminates {
cursor.eat_while(|b| b != b'\n' && b != b'\r');
} else {
cursor.eat_while(|b| b != b'\n');
}
}
fn skip_block_comment(cursor: &mut Cursor, nested: bool, silent_eof: bool) -> Result<(), LexError> {
let start = cursor.pos();
cursor.bump(); cursor.bump();
let mut depth: u32 = 1;
loop {
match (cursor.peek(), cursor.peek_nth(1)) {
(Some(b'/'), Some(b'*')) if nested => {
cursor.bump();
cursor.bump();
depth = depth
.checked_add(1)
.expect("block-comment nesting depth exceeds u32::MAX");
}
(Some(b'*'), Some(b'/')) => {
cursor.bump();
cursor.bump();
depth -= 1;
if depth == 0 {
return Ok(());
}
}
(Some(_), _) => {
cursor.bump();
}
(None, _) => {
if silent_eof {
return Ok(());
}
return Err(LexError::new(
LexErrorKind::UnterminatedBlockComment,
Span::new(start, cursor.pos()),
));
}
}
}
}
fn scan_versioned_marker(
cursor: &mut Cursor,
bound: u32,
nested: bool,
state: &mut LexState,
) -> Result<(), LexError> {
let start = cursor.pos();
cursor.advance_bytes(3);
let mut digits: u32 = 0;
while digits < 7 && cursor.peek_nth(digits).is_some_and(|b| b.is_ascii_digit()) {
digits += 1;
}
let version_len = match digits {
0..=4 => 0,
6 => 6,
_ => 5, };
let mut version: u32 = 0;
for _ in 0..version_len {
let digit = cursor.bump().expect("the digit run was just measured") - b'0';
version = version * 10 + u32::from(digit);
}
if version_len > 0 && version > bound {
return skip_discarded_region(cursor, start, nested);
}
state.versioned_region_start.get_or_insert(start);
Ok(())
}
fn skip_discarded_region(
cursor: &mut Cursor,
region_start: u32,
nested: bool,
) -> Result<(), LexError> {
loop {
match (cursor.peek(), cursor.peek_nth(1)) {
(Some(b'*'), Some(b'/')) => {
cursor.advance_bytes(2);
return Ok(());
}
(Some(b'/'), Some(b'*')) => {
skip_block_comment(cursor, nested, false)?;
}
(Some(_), _) => {
cursor.bump();
}
(None, _) => {
return Err(LexError::new(
LexErrorKind::UnterminatedBlockComment,
Span::new(region_start, cursor.pos()),
));
}
}
}
}
fn scan_word(cursor: &mut Cursor, features: &FeatureSet) -> Token {
let start = cursor.pos();
cursor
.bump_char()
.expect("dispatch routes only identifier-start characters to scan_word");
eat_identifier_continue(cursor, features);
let span = Span::new(start, cursor.pos());
let text = &cursor.src()[span.start() as usize..span.end() as usize];
let kind = lookup_keyword(text).map_or(TokenKind::Word, TokenKind::Keyword);
Token::new(kind, span)
}
fn scan_number(cursor: &mut Cursor, features: &FeatureSet) -> Result<Token, LexError> {
let start = cursor.pos();
let num = &features.numeric_literals;
let separators = num.underscore_separators;
let strict = num.reject_trailing_junk;
if let Some(radix) = radix_prefix(cursor, features) {
cursor.bump(); cursor.bump(); eat_radix_digits(cursor, radix, separators);
} else {
eat_fixed_point(cursor, features, separators, strict);
if matches!(cursor.peek(), Some(b'e' | b'E')) && exponent_follows(cursor, features) {
cursor.bump(); if matches!(cursor.peek(), Some(b'+' | b'-')) {
cursor.bump();
}
eat_decimal_digits(cursor, features, separators, strict);
}
}
if strict
&& cursor
.char_at(0)
.is_some_and(|ch| is_identifier_start(ch, features))
{
eat_identifier_continue(cursor, features);
return Err(LexError::new(
LexErrorKind::TrailingJunkAfterNumber,
Span::new(start, cursor.pos()),
));
}
Ok(Token::new(
TokenKind::Number,
Span::new(start, cursor.pos()),
))
}
fn money_follows(cursor: &Cursor, features: &FeatureSet) -> bool {
match cursor.peek_nth(1) {
Some(byte) if is_digit(byte, features) => true,
Some(b'.') => cursor
.peek_nth(2)
.is_some_and(|byte| is_digit(byte, features)),
_ => false,
}
}
fn scan_money(cursor: &mut Cursor, features: &FeatureSet) -> Token {
let start = cursor.pos();
cursor.bump(); eat_fixed_point(cursor, features, false, false);
Token::new(TokenKind::Number, Span::new(start, cursor.pos()))
}
fn eat_fixed_point(cursor: &mut Cursor, features: &FeatureSet, separators: bool, strict: bool) {
if cursor.peek() == Some(b'.') {
cursor.bump(); eat_decimal_digits(cursor, features, separators, strict);
} else {
eat_decimal_digits(cursor, features, separators, strict); if cursor.peek() == Some(b'.') {
cursor.bump(); eat_decimal_digits(cursor, features, separators, strict); }
}
}
fn eat_decimal_digits(cursor: &mut Cursor, features: &FeatureSet, separators: bool, strict: bool) {
let mut prev_was_digit = false;
loop {
match cursor.peek() {
Some(byte) if is_digit(byte, features) => {
cursor.bump();
prev_was_digit = true;
}
Some(b'_')
if separators
&& (!strict || prev_was_digit)
&& cursor.peek_nth(1).is_some_and(|b| is_digit(b, features)) =>
{
cursor.bump(); prev_was_digit = false; }
_ => return,
}
}
}
#[derive(Clone, Copy)]
enum Radix {
Hex,
Octal,
Binary,
}
impl Radix {
fn is_digit(self, byte: u8) -> bool {
match self {
Self::Hex => byte.is_ascii_hexdigit(),
Self::Octal => matches!(byte, b'0'..=b'7'),
Self::Binary => matches!(byte, b'0' | b'1'),
}
}
}
fn radix_prefix(cursor: &Cursor, features: &FeatureSet) -> Option<Radix> {
if cursor.peek() != Some(b'0') {
return None;
}
let num = &features.numeric_literals;
let radix = match cursor.peek_nth(1)? {
b'x' | b'X' if num.hex_integers => Radix::Hex,
b'o' | b'O' if num.octal_integers => Radix::Octal,
b'b' | b'B' if num.binary_integers => Radix::Binary,
_ => return None,
};
let first = cursor.peek_nth(2)?;
let opens_body = radix.is_digit(first)
|| (num.underscore_separators
&& num.radix_leading_underscore
&& first == b'_'
&& cursor.peek_nth(3).is_some_and(|b| radix.is_digit(b)));
opens_body.then_some(radix)
}
fn eat_radix_digits(cursor: &mut Cursor, radix: Radix, separators: bool) {
loop {
match cursor.peek() {
Some(byte) if radix.is_digit(byte) => {
cursor.bump();
}
Some(b'_') if separators && cursor.peek_nth(1).is_some_and(|b| radix.is_digit(b)) => {
cursor.bump();
}
_ => return,
}
}
}
fn exponent_follows(cursor: &Cursor, features: &FeatureSet) -> bool {
let mut ahead = 1;
if matches!(cursor.peek_nth(ahead), Some(b'+' | b'-')) {
ahead += 1;
}
cursor
.peek_nth(ahead)
.is_some_and(|b| is_digit(b, features))
}
fn scan_parameter(cursor: &mut Cursor, features: &FeatureSet) -> Token {
let start = cursor.pos();
match cursor.peek() {
Some(b'$') => {
cursor.bump(); if cursor.peek().is_some_and(|b| b.is_ascii_digit()) {
cursor.eat_while(|b| b.is_ascii_digit());
} else {
eat_identifier_continue(cursor, features);
}
}
Some(b':' | b'@') => {
cursor.bump(); eat_identifier_continue(cursor, features);
}
_ => {
cursor.bump(); if features.parameters.numbered_question
&& cursor.peek().is_some_and(|b| b.is_ascii_digit())
{
cursor.eat_while(|b| b.is_ascii_digit());
}
}
}
Token::new(TokenKind::Parameter, Span::new(start, cursor.pos()))
}
fn scan_positional_column(cursor: &mut Cursor) -> Token {
let start = cursor.pos();
cursor.bump(); cursor.eat_while(|b| b.is_ascii_digit());
Token::new(TokenKind::PositionalColumn, Span::new(start, cursor.pos()))
}
fn scan_stage_reference(cursor: &mut Cursor, features: &FeatureSet) -> Token {
let start = cursor.pos();
cursor.bump(); match cursor.peek() {
Some(b'~') => {
cursor.bump();
}
Some(b'%') => {
cursor.bump();
if cursor
.char_at(0)
.is_some_and(|ch| is_identifier_start(ch, features))
{
eat_identifier_continue(cursor, features);
}
}
_ => {
eat_identifier_continue(cursor, features);
while cursor.peek() == Some(b'.')
&& cursor
.char_at(1)
.is_some_and(|ch| is_identifier_start(ch, features))
{
cursor.bump(); eat_identifier_continue(cursor, features);
}
}
}
while cursor.peek() == Some(b'/') {
cursor.bump(); cursor.eat_while(|b| {
!b.is_ascii_whitespace()
&& !matches!(b, b',' | b')' | b'(' | b';' | b'=' | b'<' | b'>' | b'!')
});
}
Token::new(TokenKind::StageReference, Span::new(start, cursor.pos()))
}
fn scan_session_variable(cursor: &mut Cursor, features: &FeatureSet) -> Token {
let start = cursor.pos();
cursor.bump(); if cursor.peek() == Some(b'@') {
cursor.bump(); eat_identifier_continue(cursor, features); if cursor.peek() == Some(b'.')
&& cursor
.char_at(1)
.is_some_and(|ch| is_identifier_start(ch, features))
{
cursor.bump(); eat_identifier_continue(cursor, features);
}
} else {
eat_identifier_continue(cursor, features); }
Token::new(TokenKind::Variable, Span::new(start, cursor.pos()))
}
fn scan_quoted(cursor: &mut Cursor, scan: QuoteScan) -> Result<Token, LexError> {
let start = cursor.pos();
let token = scan_quoted_body(cursor, start, scan)?;
if token.kind == TokenKind::String && scan.backslash {
Ok(extend_string_continuations(cursor, token, scan)?.0)
} else {
Ok(token)
}
}
pub(crate) fn escape_string_segments_are_valid(text: &str) -> bool {
let bytes = text.as_bytes();
if bytes.len() < 3 || !matches!(bytes[0], b'E' | b'e') || bytes[1] != b'\'' {
return crate::ast::postgres_escape_string_is_valid(text);
}
let prefix = bytes[0];
let mut i = 1;
loop {
if i >= bytes.len() || bytes[i] != b'\'' {
return false;
}
let seg_open = i;
i += 1;
let mut closed = false;
while i < bytes.len() {
match bytes[i] {
b'\\' if i + 1 < bytes.len() => i += 2,
b'\'' if i + 1 < bytes.len() && bytes[i + 1] == b'\'' => i += 2,
b'\'' => {
let mut seg = Vec::with_capacity((i - seg_open) + 2);
seg.push(prefix);
seg.extend_from_slice(&bytes[seg_open..=i]);
let Ok(seg_text) = std::str::from_utf8(&seg) else {
return false;
};
if !crate::ast::postgres_escape_string_is_valid(seg_text) {
return false;
}
i += 1;
closed = true;
break;
}
_ => i += 1,
}
}
if !closed {
return false;
}
let mut saw_newline = false;
while i < bytes.len() {
match bytes[i] {
b'\n' | b'\r' => {
saw_newline = true;
i += 1;
}
b' ' | b'\t' | 0x0b | 0x0c => i += 1,
_ => break,
}
}
if saw_newline && i < bytes.len() && bytes[i] == b'\'' {
continue;
}
return i == bytes.len();
}
}
fn extend_string_continuations(
cursor: &mut Cursor,
mut token: Token,
scan: QuoteScan,
) -> Result<(Token, bool), LexError> {
let mut continued = false;
loop {
let save = cursor.pos();
let mut saw_newline = false;
loop {
match cursor.peek() {
Some(b'\n' | b'\r') => {
saw_newline = true;
cursor.bump();
}
Some(b' ' | b'\t' | 0x0b | 0x0c) => {
cursor.bump();
}
_ => break,
}
}
if !saw_newline {
cursor.set_pos(save);
return Ok((token, continued));
}
let close = match cursor.peek() {
Some(b'\'') => b'\'',
Some(b'"') if scan.close == b'"' => b'"',
_ => {
cursor.set_pos(save);
return Ok((token, continued));
}
};
token = scan_quoted_body(
cursor,
token.span.start(),
QuoteScan {
close,
kind: TokenKind::String,
unterminated: LexErrorKind::UnterminatedString,
backslash: scan.backslash,
allow_zero_length: false,
},
)?;
continued = true;
}
}
#[derive(Clone, Copy)]
struct QuoteScan {
close: u8,
kind: TokenKind,
unterminated: LexErrorKind,
backslash: bool,
allow_zero_length: bool,
}
fn scan_prefixed_string(
cursor: &mut Cursor,
prefix_len: u32,
close: u8,
backslash: bool,
) -> Result<(Token, bool), LexError> {
let start = cursor.pos();
cursor.advance_bytes(prefix_len); let scan = QuoteScan {
close,
kind: TokenKind::String,
unterminated: LexErrorKind::UnterminatedString,
backslash,
allow_zero_length: false,
};
let token = scan_quoted_body(cursor, start, scan)?;
if backslash {
extend_string_continuations(cursor, token, scan)
} else {
Ok((token, false))
}
}
fn scan_unicode_ident(cursor: &mut Cursor, features: &FeatureSet) -> Result<Token, LexError> {
let start = cursor.pos();
cursor.advance_bytes(2); scan_quoted_body(
cursor,
start,
QuoteScan {
close: b'"',
kind: TokenKind::QuotedIdent,
unterminated: LexErrorKind::UnterminatedQuotedIdent,
backslash: false,
allow_zero_length: features.identifier_syntax.empty_quoted_identifiers,
},
)
}
fn scan_hex_blob(cursor: &mut Cursor) -> Result<Token, LexError> {
let start = cursor.pos();
cursor.advance_bytes(2); let digits_start = cursor.pos();
cursor.eat_while(|byte| byte.is_ascii_hexdigit());
let even = (cursor.pos() - digits_start) % 2 == 0;
if even && cursor.peek() == Some(b'\'') {
cursor.bump(); return Ok(Token::new(
TokenKind::String,
Span::new(start, cursor.pos()),
));
}
cursor.eat_while(|byte| byte != b'\'');
let kind = if cursor.peek() == Some(b'\'') {
cursor.bump();
LexErrorKind::MalformedBlobLiteral
} else {
LexErrorKind::UnterminatedString
};
Err(LexError::new(kind, Span::new(start, cursor.pos())))
}
fn charset_introducer_prefix(cursor: &Cursor, features: &FeatureSet) -> Option<(u32, u8)> {
let mut len = 1; while cursor
.peek_nth(len)
.is_some_and(|byte| byte < 0x80 && is_identifier_continue(byte as char, features))
{
len += 1;
}
if len == 1 {
return None;
}
match cursor.peek_nth(len) {
Some(b'\'') => Some((len, b'\'')),
Some(b'"') if features.string_literals.double_quoted_strings => Some((len, b'"')),
_ => None,
}
}
fn scan_quoted_body(cursor: &mut Cursor, start: u32, scan: QuoteScan) -> Result<Token, LexError> {
cursor.bump(); let body_start = cursor.pos();
loop {
match cursor.peek() {
None => {
return Err(LexError::new(
scan.unterminated,
Span::new(start, cursor.pos()),
));
}
Some(b'\\') if scan.backslash => {
cursor.bump(); if cursor.bump().is_none() {
return Err(LexError::new(
scan.unterminated,
Span::new(start, cursor.pos()),
));
}
}
Some(byte) if byte == scan.close => {
if cursor.peek_nth(1) == Some(scan.close) {
cursor.bump(); cursor.bump();
} else {
let zero_length_ident = scan.kind == TokenKind::QuotedIdent
&& !scan.allow_zero_length
&& cursor.pos() == body_start;
cursor.bump(); let span = Span::new(start, cursor.pos());
return if zero_length_ident {
Err(LexError::new(
LexErrorKind::ZeroLengthDelimitedIdentifier,
span,
))
} else {
Ok(Token::new(scan.kind, span))
};
}
}
Some(_) => {
cursor.bump();
}
}
}
}
fn scan_dollar_quote(cursor: &mut Cursor, features: &FeatureSet) -> Result<Token, LexError> {
let start = cursor.pos();
let bytes = cursor.src().as_bytes();
let opener_start = start as usize;
let mut index = opener_start + 1;
if bytes
.get(index)
.is_some_and(|&b| is_dollar_tag_start(b, features))
{
index += 1;
while bytes
.get(index)
.is_some_and(|&b| is_dollar_tag_continue(b, features))
{
index += 1;
}
}
if bytes.get(index) != Some(&b'$') {
return Err(LexError::new(
LexErrorKind::StrayByte,
Span::new(start, start + 1),
));
}
let delim = &bytes[opener_start..=index];
cursor.advance_bytes(delim.len() as u32);
loop {
let rest = cursor.rest();
match rest.iter().position(|&b| b == b'$') {
Some(offset) => {
cursor.advance_bytes(offset as u32); if cursor.rest().starts_with(delim) {
cursor.advance_bytes(delim.len() as u32); return Ok(Token::new(
TokenKind::String,
Span::new(start, cursor.pos()),
));
}
cursor.bump();
}
None => {
cursor.advance_bytes(rest.len() as u32);
return Err(LexError::new(
LexErrorKind::UnterminatedDollarQuote,
Span::new(start, cursor.pos()),
));
}
}
}
}
fn scan_operator(cursor: &mut Cursor, features: &FeatureSet) -> Token {
use Operator::{
Amp, AmpAmp, Arrow, AtAt, AtGt, AtQuestion, Bang, Caret, CaretAt, Concat, Eq, EqEq, Gt,
GtEq, Hash, HashGt, HashGtGt, HashMinus, Lt, LtAt, LtEq, LtEqGt, Minus, MinusGt, MinusGtGt,
NotEq, Percent, Pipe, PipeArrow, Plus, Question, QuestionAmp, QuestionPipe, ShiftLeft,
ShiftRight, Slash, SlashSlash, Star, Tilde,
};
if features.operator_syntax.custom_operators {
return scan_custom_operator(cursor, features);
}
let start = cursor.pos();
let lead = cursor
.peek()
.expect("dispatch routes only operator-class bytes to scan_operator");
let (operator, width): (Operator, u32) = match lead {
b'+' => (Plus, 1),
b'-' if features.operator_syntax.json_arrow_operators
&& cursor.peek_nth(1) == Some(b'>') =>
{
match cursor.peek_nth(2) {
Some(b'>') => (MinusGtGt, 3),
_ => (MinusGt, 2),
}
}
b'-' => (Minus, 1),
b'*' => (Star, 1),
b'/' if features.operator_syntax.integer_divide_slash
&& cursor.peek_nth(1) == Some(b'/') =>
{
(SlashSlash, 2)
}
b'/' => (Slash, 1),
b'%' => (Percent, 1),
b'=' if features.call_syntax.named_argument && cursor.peek_nth(1) == Some(b'>') => {
(Arrow, 2)
}
b'=' if features.operator_syntax.double_equals && cursor.peek_nth(1) == Some(b'=') => {
(EqEq, 2)
}
b'=' => (Eq, 1),
b'<' => match cursor.peek_nth(1) {
Some(b'=')
if features.operator_syntax.null_safe_equals
&& cursor.peek_nth(2) == Some(b'>') =>
{
(LtEqGt, 3)
}
Some(b'=') => (LtEq, 2),
Some(b'>') => (NotEq, 2),
Some(b'<') => (ShiftLeft, 2),
Some(b'@') if features.operator_syntax.containment_operators => (LtAt, 2),
_ => (Lt, 1),
},
b'>' => match cursor.peek_nth(1) {
Some(b'=') => (GtEq, 2),
Some(b'>') => (ShiftRight, 2),
_ => (Gt, 1),
},
b'@' => match cursor.peek_nth(1) {
Some(b'>') => (AtGt, 2),
Some(b'?') => (AtQuestion, 2),
Some(b'@') => (AtAt, 2),
_ => unreachable!("the `@` dispatch arms route only `@>`/`@?`/`@@` to scan_operator"),
},
b'#' if features.operator_syntax.jsonb_operators => match cursor.peek_nth(1) {
Some(b'>') => match cursor.peek_nth(2) {
Some(b'>') => (HashGtGt, 3),
_ => (HashGt, 2),
},
Some(b'-') => (HashMinus, 2),
_ => (Hash, 1),
},
b'#' => (Hash, 1),
b'?' => match cursor.peek_nth(1) {
Some(b'|') => (QuestionPipe, 2),
Some(b'&') => (QuestionAmp, 2),
_ => (Question, 1),
},
b'!' => match cursor.peek_nth(1) {
Some(b'=') => (NotEq, 2),
_ => (Bang, 1),
},
b'|' => match cursor.peek_nth(1) {
Some(b'|') => (Concat, 2),
Some(b'>') if features.query_tail_syntax.pipe_syntax => (PipeArrow, 2),
_ => (Pipe, 1),
},
b'&' => match cursor.peek_nth(1) {
Some(b'&') => (AmpAmp, 2),
_ => (Amp, 1),
},
b'^' => match cursor.peek_nth(1) {
Some(b'@') if features.operator_syntax.starts_with_operator => (CaretAt, 2),
_ => (Caret, 1),
},
b'~' => (Tilde, 1),
_ => unreachable!("dispatch routes only operator-class bytes to scan_operator"),
};
cursor.advance_bytes(width); Token::new(
TokenKind::Operator(operator),
Span::new(start, cursor.pos()),
)
}
fn scan_custom_operator(cursor: &mut Cursor, features: &FeatureSet) -> Token {
let start = cursor.pos();
let n = custom_operator_run_len(cursor, features);
let run = &cursor.src()[start as usize..(start + n) as usize];
let (operator, width) = match known_operator_token(run, features) {
Some(operator) => (operator, n),
None if run == "=>" => (Operator::Eq, 1),
None => (Operator::Custom, n),
};
cursor.advance_bytes(width); Token::new(
TokenKind::Operator(operator),
Span::new(start, cursor.pos()),
)
}
fn is_operator_char(byte: u8, features: &FeatureSet) -> bool {
match byte {
b'#' => !features.expression_syntax.positional_column,
b'?' => !features.parameters.anonymous_question,
b'~' | b'!' | b'@' | b'^' | b'&' | b'|' | b'`' | b'+' | b'-' | b'*' | b'/' | b'%'
| b'<' | b'>' | b'=' => true,
_ => false,
}
}
fn operator_char_allows_trailing_sign(byte: u8) -> bool {
matches!(
byte,
b'~' | b'!' | b'@' | b'#' | b'^' | b'&' | b'|' | b'`' | b'?' | b'%'
)
}
fn custom_operator_run_len(cursor: &Cursor, features: &FeatureSet) -> u32 {
let mut n: u32 = 1;
while let Some(byte) = cursor.peek_nth(n) {
if !is_operator_char(byte, features) {
break;
}
let next = cursor.peek_nth(n + 1);
if (byte == b'-' && next == Some(b'-')) || (byte == b'/' && next == Some(b'*')) {
break;
}
n += 1;
}
let has_sign_char = (0..n).any(|i| {
cursor
.peek_nth(i)
.is_some_and(operator_char_allows_trailing_sign)
});
if !has_sign_char {
while n > 1 && matches!(cursor.peek_nth(n - 1), Some(b'+') | Some(b'-')) {
n -= 1;
}
}
n
}
fn known_operator_token(run: &str, features: &FeatureSet) -> Option<Operator> {
use Operator::{
Amp, AmpAmp, Arrow, AtAt, AtGt, AtQuestion, Bang, Caret, CaretAt, Concat, Eq, EqEq, Gt,
GtEq, Hash, HashGt, HashGtGt, HashMinus, Lt, LtAt, LtEq, LtEqGt, Minus, MinusGt, MinusGtGt,
NotEq, Percent, Pipe, PipeArrow, Plus, Question, QuestionAmp, QuestionPipe, ShiftLeft,
ShiftRight, Slash, SlashSlash, Star, Tilde,
};
let op = match run.as_bytes() {
b"+" => Plus,
b"-" => Minus,
b"*" => Star,
b"/" => Slash,
b"%" => Percent,
b"^" => Caret,
b"<" => Lt,
b">" => Gt,
b"=" => Eq,
b"~" => Tilde,
b"!" => Bang,
b"&" => Amp,
b"|" => Pipe,
b"#" if features.hash_bitwise_xor => Hash,
b"<=" => LtEq,
b">=" => GtEq,
b"<>" | b"!=" => NotEq,
b"<<" => ShiftLeft,
b">>" => ShiftRight,
b"||" => Concat,
b"&&" => AmpAmp,
b"?" if features.operator_syntax.jsonb_operators => Question,
b"?|" if features.operator_syntax.jsonb_operators => QuestionPipe,
b"?&" if features.operator_syntax.jsonb_operators => QuestionAmp,
b"@?" if features.operator_syntax.jsonb_operators => AtQuestion,
b"@@" if features.operator_syntax.jsonb_operators => AtAt,
b"#>" if features.operator_syntax.jsonb_operators => HashGt,
b"#>>" if features.operator_syntax.jsonb_operators => HashGtGt,
b"#-" if features.operator_syntax.jsonb_operators => HashMinus,
b"@>" if features.operator_syntax.containment_operators => AtGt,
b"<@" if features.operator_syntax.containment_operators => LtAt,
b"->" if features.operator_syntax.json_arrow_operators => MinusGt,
b"->>" if features.operator_syntax.json_arrow_operators => MinusGtGt,
b"=>" if features.call_syntax.named_argument => Arrow,
b"|>" if features.query_tail_syntax.pipe_syntax => PipeArrow,
b"==" if features.operator_syntax.double_equals => EqEq,
b"//" if features.operator_syntax.integer_divide_slash => SlashSlash,
b"<=>" if features.operator_syntax.null_safe_equals => LtEqGt,
b"^@" if features.operator_syntax.starts_with_operator => CaretAt,
_ => return None,
};
Some(op)
}
fn scan_punctuation(cursor: &mut Cursor, features: &FeatureSet) -> Token {
use Punctuation::{
Colon, Comma, Dot, DoubleColon, LBrace, LBracket, LParen, RBrace, RBracket, RParen,
Semicolon,
};
let start = cursor.pos();
let byte = cursor
.peek()
.expect("dispatch routes only punctuation-class bytes to scan_punctuation");
if byte == b':' && cursor.peek_nth(1) == Some(b':') {
cursor.bump();
cursor.bump();
return Token::new(
TokenKind::Punctuation(DoubleColon),
Span::new(start, cursor.pos()),
);
}
if byte == b':'
&& (features.call_syntax.named_argument || features.session_variables.variable_assignment)
&& cursor.peek_nth(1) == Some(b'=')
{
cursor.bump();
cursor.bump();
return Token::new(
TokenKind::Operator(Operator::ColonEquals),
Span::new(start, cursor.pos()),
);
}
cursor.bump();
let punctuation = match byte {
b'(' => LParen,
b')' => RParen,
b',' => Comma,
b';' => Semicolon,
b'.' => Dot,
b'[' => LBracket,
b']' => RBracket,
b'{' => LBrace,
b'}' => RBrace,
b':' => Colon,
_ => unreachable!("dispatch routes only punctuation-class bytes to scan_punctuation"),
};
Token::new(
TokenKind::Punctuation(punctuation),
Span::new(start, cursor.pos()),
)
}
fn is_identifier_start(ch: char, features: &FeatureSet) -> bool {
if ch.is_ascii() {
features.has_byte_class(ch as u8, CLASS_IDENTIFIER_START)
} else {
match features.identifier_syntax.non_ascii {
crate::ast::dialect::NonAsciiIdentifierSyntax::UnicodeAlphanumeric => {
ch.is_alphabetic()
}
crate::ast::dialect::NonAsciiIdentifierSyntax::Any => true,
}
}
}
fn is_identifier_continue(ch: char, features: &FeatureSet) -> bool {
if ch.is_ascii() {
features.has_byte_class(ch as u8, CLASS_IDENTIFIER_CONTINUE)
} else {
match features.identifier_syntax.non_ascii {
crate::ast::dialect::NonAsciiIdentifierSyntax::UnicodeAlphanumeric => {
ch.is_alphanumeric()
}
crate::ast::dialect::NonAsciiIdentifierSyntax::Any => true,
}
}
}
fn eat_identifier_continue(cursor: &mut Cursor, features: &FeatureSet) {
let dollar = features.identifier_syntax.dollar_in_identifiers;
while let Some(byte) = cursor.peek() {
if byte < 0x80 {
if !(is_identifier_continue(byte as char, features) || (dollar && byte == b'$')) {
return;
}
cursor.bump();
} else {
let ch = cursor
.char_at(0)
.expect("a non-ASCII lead byte begins a valid UTF-8 char in &str source");
if !is_identifier_continue(ch, features) {
return;
}
cursor.advance_bytes(ch.len_utf8() as u32); }
}
}
fn is_dollar_tag_start(byte: u8, features: &FeatureSet) -> bool {
features.has_byte_class(byte, CLASS_IDENTIFIER_START) || byte >= 0x80
}
fn is_dollar_tag_continue(byte: u8, features: &FeatureSet) -> bool {
features.has_byte_class(byte, CLASS_IDENTIFIER_CONTINUE)
}
fn is_digit(byte: u8, features: &FeatureSet) -> bool {
features.has_byte_class(byte, CLASS_DIGIT)
}
fn opening_identifier_quote(features: &FeatureSet, byte: u8) -> Option<u8> {
features.identifier_quotes.iter().find_map(|style| {
let (open, close) = (style.open(), style.close());
(open.is_ascii() && open as u8 == byte && close.is_ascii()).then_some(close as u8)
})
}