orql 0.1.0

//! Provides [Scanner] to parse Oracle specific SQL into a lazy
//! iterator over individual tokens.
//!
//! Usage example:
//!
//! ```rust
//! use orql::scanner::*;
//!
//! let mut scan = Scanner::new("select * from dual;");
//! assert_eq!(scan.next(), Some(Ok(Token {
//!     ttype: TokenType::Keyword(Keyword::SELECT),
//!     loc: (1, 1).into(),
//! })));
//! assert_eq!(scan.next(), Some(Ok(Token {
//!     ttype: TokenType::Star,
//!     loc: (1, 8).into(),
//! })));
//! // ...
//! ```

use std::{
    borrow::Cow,
    iter::Peekable,
    ops::Range,
    str::{Chars, FromStr},
};

mod error;
mod keywords;
mod text;
mod tokens;

pub use error::*;
pub(crate) use keywords::BuiltIn;
pub use keywords::{Keyword, Reserved};
pub use text::*;
pub use tokens::*;

#[cfg(test)]
mod tests;

/// An iterator over tokens from a lazily parsed SQL string.
pub struct Scanner<'s> {
    source: &'s str,
    /// a stream of characters from `source`
    chars: Peekable<Chars<'s>>,
    /// location of the next character, ie. `self.chars.next()`
    next_loc: Location,
    /// offset into `source` of the next character, ie. `self.chars.next()`,
    /// `>= source.len()` if reached EOF
    next_pos: usize,
    /// a pushed back state and token to be served by `self.chars.next()`;
    /// note: while `self.pushed_back.is_none()`, the `chars` iterator might
    /// be out of sync with the scanner's direct state
    pushed_back: Option<PushedBack<'s>>,
}

/// Temporary state pushed back and to be restored later
struct PushedBack<'s> {
    token: Option<Result<Token<'s>>>,
    next_loc: Location,
}

// ----------------------------------------------------------------------------

impl<'s> Scanner<'s> {
    pub fn new(source: &'s str) -> Self {
        Self {
            source,
            chars: source.chars().peekable(),
            next_loc: Location { line: 1, col: 1 },
            next_pos: 0,
            pushed_back: None,
        }
    }

    /// Retrieves the underlying source string being processed.
    pub(crate) fn source(&self) -> &'s str {
        self.source
    }

    /// Retrieves the current position within the underlying `source` text
    /// (where `position` is a zero based index into `source`.)  This is the
    /// position where the scanner will start finding the next token. The
    /// initial position is zero. After hitting the "end of the source" the
    /// position will be equal to the source's length.
    pub(crate) fn position(&self) -> usize {
        self.next_pos
    }

    /// Retrieves the location of the scanner is its `source` effectively
    /// telling the approximate location of the next token.
    pub fn location(&self) -> Location {
        self.next_loc
    }

    /// Turns this scanner into one that can "peek" at the next token (without
    /// consuming it), just like with [std::iter::Peekable].  The returned
    /// "peekable", however, will allow to access the underlying scanner.
    pub fn peekable(self) -> PeekableScanner<'s> {
        PeekableScanner {
            inner: self,
            peeked: None,
        }
    }
}

impl<'s> Iterator for Scanner<'s> {
    type Item = Result<Token<'s>>;

    fn next(&mut self) -> Option<Self::Item> {
        if let Some(state) = self.pushed_back.take() {
            self.next_loc = state.next_loc;
            return state.token;
        }

        loop {
            let token_pos = self.next_pos;
            let token_loc = self.next_loc;

            macro_rules! emit_token {
                ($ttype:expr) => {
                    return Some(Ok(Token {
                        ttype: $ttype,
                        loc: token_loc,
                    }))
                };
            }
            macro_rules! expect_char {
                ($char:literal, $ttype:expr) => {
                    // ~ do not consume the peeked char upon an error;
                    // maybe it's the start of a valid, next token
                    match self.peek_char() {
                        Some($char) => {
                            self.consume_char_and_col();
                            emit_token!($ttype)
                        }
                        Some(c) => {
                            return Some(Err(Error {
                                etype: ErrorType::InvalidChar {
                                    char: c,
                                    expected: Some(concat!("a '", $char, "'")),
                                },
                                loc: self.next_loc,
                            }));
                        }
                        None => {
                            return Some(Err(Error {
                                etype: ErrorType::UnexpectedEof {
                                    expected: Some(concat!("a '", $char, "'")),
                                },
                                loc: self.next_loc,
                            }));
                        }
                    }
                };
            }
            match self.next_char()? {
                '\r' => {
                    if matches!(self.peek_char(), Some('\n')) {
                        self.consume_char();
                    }
                    self.next_loc.line += 1;
                    self.next_loc.col = 1;
                }
                '\n' => {
                    self.next_loc.line += 1;
                    self.next_loc.col = 1;
                }
                c => {
                    self.next_loc.col += 1;
                    match c {
                        ' ' | '\t' => {}
                        '(' => emit_token!(TokenType::LeftParen),
                        ')' => emit_token!(TokenType::RightParen),
                        '.' => {
                            if matches!(self.peek_char(), Some('0'..='9')) {
                                return Some(self.consume_fraction(token_loc, token_pos));
                            } else {
                                emit_token!(TokenType::Dot);
                            };
                        }
                        '0'..='9' => return Some(self.consume_number(token_loc, token_pos)),
                        ',' => emit_token!(TokenType::Comma),
                        ';' => emit_token!(TokenType::Semicolon),
                        '+' => emit_token!(TokenType::Plus),
                        '*' => emit_token!(TokenType::Star),
                        '|' => expect_char!('|', TokenType::PipePipe),
                        '?' => emit_token!(TokenType::QuestionMark),
                        ':' => return Some(self.consume_placeholder(token_loc)),
                        '@' => emit_token!(TokenType::At),
                        '=' => {
                            emit_token!(match self.peek_char() {
                                Some('>') => {
                                    self.consume_char_and_col();
                                    TokenType::EqualGreater
                                }
                                _ => TokenType::Equal,
                            })
                        }
                        '<' => {
                            emit_token!(match self.peek_char() {
                                Some('=') => {
                                    self.consume_char_and_col();
                                    TokenType::LessEqual
                                }
                                Some('>') => {
                                    self.consume_char_and_col();
                                    TokenType::LessGreater
                                }
                                _ => TokenType::Less,
                            });
                        }
                        '!' => expect_char!('=', TokenType::BangEqual),
                        '>' => {
                            emit_token!(if matches!(self.peek_char(), Some('=')) {
                                self.consume_char_and_col();
                                TokenType::GreaterEqual
                            } else {
                                TokenType::Greater
                            });
                        }
                        '/' => {
                            if matches!(self.peek_char(), Some('*')) {
                                self.consume_char_and_col();
                                return Some(
                                    self.consume_block_comment(self.next_loc.with_cols_removed(2)),
                                );
                            } else {
                                emit_token!(TokenType::Slash)
                            }
                        }
                        '-' => {
                            if matches!(self.peek_char(), Some('-')) {
                                self.consume_char();
                                return Some(Ok(self.consume_line_comment(token_loc)));
                            } else {
                                emit_token!(TokenType::Minus)
                            }
                        }
                        '^' => expect_char!('=', TokenType::CaretEqual),
                        '\'' => return Some(self.consume_text(token_loc, NationalStyle::None)),
                        '"' => return Some(self.consume_quoted_ident(token_loc).map(Into::into)),
                        'n' | 'N' => match self.peek_char() {
                            Some('\'') => {
                                self.consume_char_and_col();
                                return Some(self.consume_text(token_loc, NationalStyle::National));
                            }
                            Some('q') | Some('Q') => {
                                self.consume_char_and_col();
                                return Some(if matches!(self.peek_char(), Some('\'')) {
                                    self.consume_char_and_col();
                                    self.consume_quoted_text(token_loc, NationalStyle::National)
                                } else {
                                    self.consume_ident_or_keyword(token_loc, token_pos)
                                        .map(Into::into)
                                });
                            }
                            _ => {
                                return Some(
                                    self.consume_ident_or_keyword(token_loc, token_pos)
                                        .map(Into::into),
                                );
                            }
                        },
                        'q' | 'Q' => {
                            return Some(if matches!(self.peek_char(), Some('\'')) {
                                self.consume_char_and_col();
                                self.consume_quoted_text(token_loc, NationalStyle::None)
                            } else {
                                self.consume_ident_or_keyword(token_loc, token_pos)
                                    .map(Into::into)
                            });
                        }
                        c if c.is_alphabetic() => {
                            return Some(
                                self.consume_ident_or_keyword(token_loc, token_pos)
                                    .map(Into::into),
                            );
                        }
                        _ => {
                            return Some(Err(Error {
                                etype: ErrorType::InvalidChar {
                                    char: c,
                                    expected: None,
                                },
                                loc: self.next_loc,
                            }));
                        }
                    }
                }
            }
        }
    }
}

impl<'s> Scanner<'s> {
    fn next_char(&mut self) -> Option<char> {
        let c = self.chars.next()?;
        self.next_pos += c.len_utf8();
        Some(c)
    }

    fn peek_char(&mut self) -> Option<char> {
        self.chars.peek().copied()
    }

    /// Consumes the next char
    fn consume_char(&mut self) {
        _ = self.next_char();
    }

    /// Consume the next char assuming it is not a newline; increments `self.next_loc.col`.
    fn consume_char_and_col(&mut self) {
        _ = self.next_char();
        self.next_loc.col += 1;
    }

    fn curr_lexem(&self, pos: usize) -> &'s str {
        &self.source[pos..self.next_pos]
    }

    /// Scans a number assuming a digit has been identified (and consumed) at
    /// `token_pos` / `token_loc`.
    fn consume_number(&mut self, token_loc: Location, token_pos: usize) -> Result<Token<'s>> {
        while matches!(self.peek_char(), Some('0'..='9')) {
            self.consume_char_and_col();
        }
        if matches!(self.peek_char(), Some('.')) {
            self.consume_char_and_col();
            self.consume_fraction(token_loc, token_pos)
        } else {
            Ok(Token {
                ttype: TokenType::Integer(self.curr_lexem(token_pos)),
                loc: token_loc,
            })
        }
    }

    fn consume_fraction(&mut self, token_loc: Location, token_pos: usize) -> Result<Token<'s>> {
        // ~ optional
        while matches!(self.peek_char(), Some('0'..='9')) {
            self.consume_char_and_col();
        }
        // ~ optional
        if matches!(self.peek_char(), Some('e') | Some('E')) {
            self.consume_char_and_col();
            // ~ optional
            if matches!(self.peek_char(), Some('+') | Some('-')) {
                self.consume_char_and_col();
            }
            // ~ at least one digit expected
            match self.peek_char() {
                None => {
                    return Err(Error {
                        etype: ErrorType::UnexpectedEof {
                            expected: Some("a digit"),
                        },
                        loc: self.next_loc,
                    });
                }
                Some('0'..='9') => {
                    self.consume_char_and_col();
                }
                Some(c) => {
                    return Err(Error {
                        etype: ErrorType::InvalidChar {
                            char: c,
                            expected: Some("a digit"),
                        },
                        loc: self.next_loc,
                    });
                }
            }
            while matches!(self.peek_char(), Some('0'..='9')) {
                self.consume_char_and_col();
            }
        };
        // ~ optional
        match self.peek_char() {
            Some('f') | Some('F') => {
                self.consume_char_and_col();
            }
            Some('d') | Some('D') => {
                self.consume_char_and_col();
            }
            _ => {}
        }
        Ok(Token {
            ttype: TokenType::Float(self.curr_lexem(token_pos)),
            loc: token_loc,
        })
    }

    /// Reads until "*/"
    fn consume_block_comment(&mut self, token_loc: Location) -> Result<Token<'s>> {
        let start_pos = self.next_pos;
        while let Some(c) = self.next_char() {
            match c {
                '\r' => {
                    if matches!(self.peek_char(), Some('\n')) {
                        self.consume_char();
                    }
                    self.next_loc.line += 1;
                    self.next_loc.col = 1;
                }
                '\n' => {
                    self.next_loc.line += 1;
                    self.next_loc.col = 1;
                }
                c => {
                    self.next_loc.col += 1;
                    if c == '*' && matches!(self.peek_char(), Some('/')) {
                        self.consume_char_and_col();
                        return Ok(Token {
                            ttype: TokenType::Comment(Comment(
                                // ~ do not include the terminating "*/"
                                &self.source[start_pos..self.next_pos - 2],
                                CommentStyle::Block,
                            )),
                            loc: token_loc,
                        });
                    }
                }
            }
        }
        Err(Error {
            etype: ErrorType::UnexpectedEof { expected: None },
            loc: self.next_loc,
        })
    }

    /// Reads until a newline or end-of-file
    fn consume_line_comment(&mut self, token_loc: Location) -> Token<'s> {
        let (start_pos, mut newline_len) = (self.next_pos, 0);
        while let Some(c) = self.next_char() {
            match c {
                '\r' => {
                    if matches!(self.peek_char(), Some('\n')) {
                        self.consume_char();
                        newline_len = 2;
                    } else {
                        newline_len = 1;
                    }
                    self.next_loc.line += 1;
                    self.next_loc.col = 1;
                    break;
                }
                '\n' => {
                    self.next_loc.line += 1;
                    self.next_loc.col = 1;
                    newline_len = 1;
                    break;
                }
                _ => {
                    self.next_loc.col += 1;
                }
            }
        }
        Token {
            ttype: TokenType::Comment(Comment(
                &self.source[start_pos..self.next_pos - newline_len],
                CommentStyle::Line,
            )),
            loc: token_loc,
        }
    }

    /// Reads a "plain" string literal, assuming the opening quote is already consumed.
    fn consume_text(&mut self, token_loc: Location, encoding: NationalStyle) -> Result<Token<'s>> {
        // first: loop until hitting the end of the string and then _return_
        // the identified slice; however, when hitting a `''` _break_, make a
        // copy of the the so-far identified string part and continue
        // accumlating into a buffer with `''` replaced by one `'`.
        let start_pos = self.next_pos;
        while let Some(c) = self.next_char() {
            match c {
                '\r' => {
                    if matches!(self.peek_char(), Some('\n')) {
                        self.consume_char();
                    }
                    self.next_loc.line += 1;
                    self.next_loc.col = 1;
                }
                '\n' => {
                    self.next_loc.line += 1;
                    self.next_loc.col = 1;
                }
                c => {
                    self.next_loc.col += 1;
                    if c == '\'' {
                        if matches!(self.peek_char(), Some('\'')) {
                            self.consume_char_and_col();
                            break;
                        } else {
                            let s = self.curr_lexem(start_pos); // ~ includes the consumed `'`
                            return Ok(Token {
                                ttype: TokenType::Text(
                                    Text::Regular(Cow::Borrowed(&s[..s.len() - 1])),
                                    encoding,
                                ),
                                loc: token_loc,
                            });
                        }
                    }
                }
            }
        }
        // ~ at this point the last consumed part of the string was a "''"
        // which we need to turn into a single "'" ... make a copy of the
        // consumed part (without the second quote) and continue accumulating
        // while producing only one quote for every "''".
        let mut accum = String::new();
        {
            let s = self.curr_lexem(start_pos);
            accum.push_str(&s[..s.len() - 1]);
        };
        while let Some(c) = self.next_char() {
            match c {
                '\r' => {
                    if matches!(self.peek_char(), Some('\n')) {
                        self.consume_char();
                    }
                    self.next_loc.line += 1;
                    self.next_loc.col = 1;
                    accum.push(c);
                }
                '\n' => {
                    self.next_loc.line += 1;
                    self.next_loc.col = 1;
                    accum.push(c);
                }
                c => {
                    self.next_loc.col += 1;
                    match c {
                        '\'' => {
                            if matches!(self.peek_char(), Some('\'')) {
                                self.consume_char_and_col();
                                accum.push('\'');
                            } else {
                                return Ok(Token {
                                    ttype: TokenType::Text(
                                        Text::Regular(Cow::Owned(accum)),
                                        encoding,
                                    ),
                                    loc: token_loc,
                                });
                            }
                        }
                        c => accum.push(c),
                    }
                }
            }
        }
        Err(Error {
            etype: ErrorType::UnterminatedString,
            loc: token_loc,
        })
    }

    /// Reads a "quote delimited" string literal, assuming the opening quote
    /// is already consumed.
    ///
    /// See <https://docs.oracle.com/en/database/oracle/oracle-database/21/sqlrf/Literals.html>
    fn consume_quoted_text(
        &mut self,
        token_loc: Location,
        encoding: NationalStyle,
    ) -> Result<Token<'s>> {
        let start_pos = self.next_pos;
        let error_loc = self.next_loc;
        match self.next_char() {
            Some(c @ ' ') | Some(c @ '\t') | Some(c @ '\r') | Some(c @ '\n') => {
                // ~ consume until the closing "'" or "{c}'" (whichever comes
                // first) to allow consuming further tokens after having
                // reported the error
                let mut q = c;
                loop {
                    match q {
                        '\r' => {
                            if matches!(self.peek_char(), Some('\n')) {
                                self.consume_char();
                            }
                            self.next_loc.line += 1;
                            self.next_loc.col = 1;
                        }
                        '\n' => {
                            self.next_loc.line += 1;
                            self.next_loc.col = 1;
                        }
                        nc => {
                            self.next_loc.col += 1;
                            if nc == '\'' {
                                break;
                            }
                        }
                    }
                    match self.next_char() {
                        Some(c) => q = c,
                        None => break,
                    };
                }
                Err(Error {
                    etype: ErrorType::InvalidChar {
                        char: c,
                        expected: Some("a non-whitespace character"),
                    },
                    loc: error_loc,
                })
            }
            Some(start_delim) => {
                self.next_loc.col += 1;

                let end_delim = match start_delim {
                    '{' => '}',
                    '[' => ']',
                    '<' => '>',
                    '(' => ')',
                    c => c,
                };
                // ~ no escaping for quote delimited strings
                while let Some(c) = self.next_char() {
                    match c {
                        '\r' => {
                            if matches!(self.peek_char(), Some('\n')) {
                                self.consume_char();
                            }
                            self.next_loc.line += 1;
                            self.next_loc.col = 1;
                        }
                        '\n' => {
                            self.next_loc.line += 1;
                            self.next_loc.col = 1;
                        }
                        c => {
                            self.next_loc.col += 1;
                            if c == end_delim && matches!(self.peek_char(), Some('\'')) {
                                let s = self.curr_lexem(start_pos);
                                self.consume_char_and_col(); // the peeked "'"
                                return Ok(Token {
                                    ttype: TokenType::Text(
                                        Text::Quoted(QuotedText::new_unchecked(s)),
                                        encoding,
                                    ),
                                    loc: token_loc,
                                });
                            }
                        }
                    }
                }
                Err(Error {
                    etype: ErrorType::UnterminatedString,
                    loc: token_loc,
                })
            }
            None => Err(Error {
                etype: ErrorType::UnexpectedEof {
                    expected: Some("a non-whitespace quote delimiter character"),
                },
                loc: error_loc,
            }),
        }
    }

    /// Reads a quoted identifier (ie. until encountering a double quote)
    /// assuming the initial double quote has been consumed.
    fn consume_quoted_ident(&mut self, token_loc: Location) -> Result<ScannedIdent<'s>> {
        let start_pos = self.next_pos;
        while let Some(c) = self.next_char() {
            match c {
                '\r' => {
                    if matches!(self.peek_char(), Some('\n')) {
                        self.consume_char();
                    }
                    self.next_loc.line += 1;
                    self.next_loc.col = 1;
                }
                '\n' => {
                    self.next_loc.line += 1;
                    self.next_loc.col = 1;
                }
                '\0' => {
                    let error_loc = self.next_loc;
                    self.next_loc.col += 1;
                    // ~ consume until the next double quote so that we can
                    // continue serving more tokens
                    while let Some(c) = self.next_char() {
                        match c {
                            '\r' => {
                                if matches!(self.peek_char(), Some('\n')) {
                                    self.consume_char();
                                }
                                self.next_loc.line += 1;
                                self.next_loc.col = 1;
                            }
                            '\n' => {
                                self.next_loc.line += 1;
                                self.next_loc.col = 1;
                            }
                            c => {
                                self.next_loc.col += 1;
                                if c == '"' {
                                    break;
                                }
                            }
                        }
                    }
                    return Err(Error {
                        etype: ErrorType::InvalidChar {
                            char: '\0',
                            expected: None,
                        },
                        loc: error_loc,
                    });
                }
                '"' => {
                    self.next_loc.col += 1;
                    let s = &self.source[start_pos..self.next_pos - 1];
                    if s.is_empty() {
                        return Err(Error {
                            etype: ErrorType::EmptyIdent,
                            loc: token_loc,
                        });
                    } else {
                        return Ok(ScannedIdent(Ident(s, QuoteStyle::Quoted), token_loc));
                    }
                }
                _ => {
                    self.next_loc.col += 1;
                }
            }
        }
        Err(Error {
            etype: ErrorType::UnterminatedIdent,
            loc: token_loc,
        })
    }

    /// Reads a plain / non-quoted identifier (or keyword) starting at `token_pos` assuming the
    /// character at `token_pos` is alphabetic, hence a valid identifier start.
    fn consume_ident_or_keyword(
        &mut self,
        token_loc: Location,
        token_pos: usize,
    ) -> Result<IdentOrKeyword<'s>> {
        while let Some(c) = self.peek_char() {
            if c.is_alphanumeric() || c == '_' || c == '$' || c == '#' {
                self.consume_char_and_col();
            } else {
                break;
            }
        }
        let ident = self.curr_lexem(token_pos);
        Ok(match Keyword::from_str(ident) {
            Ok(kw) => IdentOrKeyword::Keyword(kw, token_loc),
            Err(_) => {
                IdentOrKeyword::Ident(ScannedIdent(Ident(ident, QuoteStyle::None), token_loc))
            }
        })
    }

    /// Reads a (mandatory) placeholder identifier. The leading ':' or '?' is assumed to be consumed already.
    ///
    /// See <https://docs.oracle.com/en/database/oracle/oracle-database/21/sqlrf/Placeholder-Expressions.html>
    fn consume_placeholder(&mut self, token_loc: Location) -> Result<Token<'s>> {
        let start_pos = self.next_pos;
        let start_loc = self.next_loc;
        let ident = match self.peek_char() {
            Some(c) if c.is_alphabetic() => {
                self.consume_char_and_col();
                self.consume_ident_or_keyword(token_loc, start_pos)?
            }
            Some('"') => {
                self.consume_char_and_col();
                IdentOrKeyword::Ident(self.consume_quoted_ident(token_loc)?)
            }
            Some(c) => {
                return Err(Error {
                    etype: ErrorType::InvalidChar {
                        char: c,
                        expected: Some("an identifier starting with an alphabetic character"),
                    },
                    loc: self.next_loc,
                });
            }
            None => {
                return Err(Error {
                    etype: ErrorType::UnexpectedEof {
                        expected: Some("an identifier starting with an alphabetic character"),
                    },
                    loc: self.next_loc,
                });
            }
        };
        Ok(match ident {
            IdentOrKeyword::Ident(ident) => Token {
                ttype: TokenType::Placeholder(ident.0),
                loc: ident.1,
            },
            IdentOrKeyword::Keyword(kw, _) => {
                debug_assert!(self.pushed_back.is_none());

                // ~ keywords are not allowed as placeholders, so we push back
                // the identified keyword for the next call to self.next() and
                // serve an error right now for the invalid placeholder,
                // essentially consuming the placeholders beginninig
                // discriminator
                let _ = self.pushed_back.insert(PushedBack {
                    token: Some(Ok(Token {
                        ttype: TokenType::Keyword(kw),
                        loc: start_loc,
                    })),
                    next_loc: self.next_loc,
                });
                // we need to restore `self.next_loc` for correct usage with
                // tokens ends; the current
                self.next_loc = start_loc;

                return Err(Error {
                    etype: ErrorType::InvalidPlaceholder {
                        details: "placeholder name must not be a keyword",
                    },
                    loc: token_loc,
                });
            }
        })
    }
}

struct ScannedIdent<'s>(Ident<'s>, Location);

impl<'s> From<ScannedIdent<'s>> for Token<'s> {
    fn from(ScannedIdent(ident, location): ScannedIdent<'s>) -> Self {
        // XXX could we merge this lookup with the lookup for keywords? both have distinct variants.
        let reserved = Reserved::lookup_ident(&ident).ok();
        Self {
            ttype: TokenType::Identifier(ident, reserved),
            loc: location,
        }
    }
}

enum IdentOrKeyword<'s> {
    Ident(ScannedIdent<'s>),
    Keyword(Keyword, Location),
}

impl<'s> From<IdentOrKeyword<'s>> for Token<'s> {
    fn from(value: IdentOrKeyword<'s>) -> Self {
        match value {
            IdentOrKeyword::Ident(ident) => ident.into(),
            IdentOrKeyword::Keyword(keyword, location) => Self {
                ttype: TokenType::Keyword(keyword),
                loc: location,
            },
        }
    }
}

// ----------------------------------------------------------------------------

/// A [Scanner] wrapper able to hold up to one look-ahead token allowing
/// temporary access to the wrapped scanner at the same time.
///
/// See [`Scanner::peekable`]
pub struct PeekableScanner<'s> {
    inner: Scanner<'s>,
    peeked: Option<Option<Result<Token<'s>>>>,
}

impl<'s> PeekableScanner<'s> {
    #[inline]
    pub fn inner(&self) -> &Scanner<'s> {
        &self.inner
    }

    /// A convenience shortcut for `self.inner().location()` deliver the
    /// underlying scanner's current location in its `source`. This might
    /// represent a point past the current [Self::peek]ed token.
    pub fn location(&self) -> Location {
        self.inner.location()
    }

    /// Extracts a slice from the underlying source.
    pub(crate) fn source_range(&self, range: Range<usize>) -> &'s str {
        let s = self.source();
        if range.start >= range.end || range.start >= s.len() {
            ""
        } else {
            &s[range.start..range.end.min(s.len())]
        }
    }

    /// Retrieves the underlying source string being processed.
    pub(crate) fn source(&self) -> &'s str {
        self.inner.source()
    }

    /// Retrieves the underlying scanner's current position - a zero based
    /// index into the underlying source slice.
    ///
    /// Mind that the returned position will be already beyond the peeked
    /// token, possibly at the start of the next token following the peeked
    /// one.
    pub(crate) fn position(&self) -> usize {
        self.inner.position()
    }

    #[inline]
    pub fn peek(&mut self) -> Option<&Result<Token<'s>>> {
        self.peeked
            .get_or_insert_with(|| self.inner.next())
            .as_ref()
    }
}

impl<'s> Iterator for PeekableScanner<'s> {
    type Item = Result<Token<'s>>;

    #[inline]
    fn next(&mut self) -> Option<Self::Item> {
        match self.peeked.take() {
            Some(v) => v,
            None => self.inner.next(),
        }
    }
}