dbt_antlr4/

lexer.rs

//! Lexer implementation
use std::borrow::Cow::Borrowed;
use std::borrow::{Borrow, Cow};
use std::cell::Cell;

use std::rc::Rc;

use crate::char_stream::CharStream;
use crate::error_listener::{ConsoleErrorListener, ErrorListener};
use crate::errors::ANTLRError;
use crate::int_stream::IntStream;
use crate::lexer_atn_simulator::{ILexerATNSimulator, LexerATNSimulator};

use crate::recognizer::{Actions, Recognizer};
use crate::rule_context::EmptyRuleNode;
use crate::token::TOKEN_INVALID_TYPE;
use crate::token_factory::TokenFactory;
use crate::token_source::TokenSource;
use std::ops::{Deref, DerefMut};

///  Lexer functionality required by `LexerATNSimulator` to work properly
pub trait Lexer<'input, 'arena, Input, TF>:
    TokenSource<'input, 'arena, TF> + Recognizer<'input, 'arena>
where
    'input: 'arena,
    Input: CharStream<'input>,
    TF: TokenFactory<'input, 'arena> + 'arena,
{
    /// Same as `TokenStream::get_input_stream` but returns concrete type instance
    /// important for proper inlining in hot code of `LexerATNSimulator`
    fn input(&mut self) -> &mut Input;
    /// Sets channel where current token will be pushed
    ///
    /// By default two channels are available:
    ///  - `LEXER_DEFAULT_TOKEN_CHANNEL`
    ///  - `LEXER_HIDDEN`
    fn set_channel(&mut self, v: i32);

    /// Pushes current mode to internal mode stack and sets `m` as current lexer mode
    /// `pop_mode should be used to recover previous mode
    fn push_mode(&mut self, m: usize);

    /// Pops mode from internal mode stack
    fn pop_mode(&mut self) -> Option<usize>;

    /// Sets type of the current token
    /// Called from action to override token that will be emitted by lexer
    fn set_type(&mut self, t: i32);

    /// Sets lexer mode discarding current one
    fn set_mode(&mut self, m: usize);

    /// Used to informs lexer that it should consider next token as a continuation of the current one
    fn more(&mut self);

    /// Tells lexer to completely ignore and not emit current token.
    fn skip(&mut self);

    #[doc(hidden)]
    fn reset(&mut self);

    #[doc(hidden)]
    fn get_interpreter(&self) -> Option<&LexerATNSimulator>;
}

/// **! Usually generated by ANTLR !**
///
/// This trait combines everything that can be used to extend Lexer behavior
pub trait LexerRecog<'input, 'arena, TF, R>: Actions<'input, 'arena, R> + Sized + 'static
where
    'input: 'arena,
    TF: TokenFactory<'input, 'arena> + 'arena,
    R: Recognizer<'input, 'arena>,
{
    /// Callback to extend emit behavior
    fn before_emit(_lexer: &mut R) {}

    fn get_rule_names(&self) -> &'static [&'static str];

    fn get_literal_names(&self) -> &[Option<&str>];

    fn get_symbolic_names(&self) -> &[Option<&str>];

    fn get_grammar_file_name(&self) -> &'static str;

    fn get_atn_simulator(&self) -> LexerATNSimulator;
}

/// Default implementation of Lexer
///
/// Public fields in this struct are intended to be used by embedded actions
#[allow(missing_docs)]
pub struct BaseLexer<'input, 'arena, Ext, Input, TF>
where
    'input: 'arena,
    Ext: LexerRecog<'input, 'arena, TF, Self> + 'static,
    Input: CharStream<'input>,
    TF: TokenFactory<'input, 'arena> + 'arena,
{
    /// `LexerATNSimulator` instance of this lexer
    pub interpreter: Option<Box<LexerATNSimulator>>,
    /// `CharStream` used by this lexer
    pub input: Option<Input>,
    recog: Ext,

    factory: TF,

    error_listeners: Vec<Box<dyn ErrorListener<'input, 'arena, Self>>>,

    pub token_start_char_index: isize,
    pub token_start_line: u32,
    pub token_start_column: i32,
    current_pos: Rc<LexerPosition>,
    /// Overrides token type emitted by lexer for current token
    pub token_type: i32,
    /// Make it `Some` to override token that is currently being generated by lexer
    pub token: Option<&'arena mut TF::Tok>,
    hit_eof: bool,
    /// Channel lexer is currently assigning tokens to
    pub channel: i32,
    /// stack of modes, which is used for pushMode,popMode lexer actions
    pub mode_stack: Vec<usize>,
    /// Mode lexer is currently in
    pub mode: usize,
    /// Make it `Some` to override text for token that is currently being generated by lexer
    pub text: Option<String>,
}

#[derive(Debug)]
pub(crate) struct LexerPosition {
    pub(crate) line: Cell<u32>,
    pub(crate) char_position_in_line: Cell<i32>,
}

impl<'input, 'arena, Ext, Input, TF> Deref for BaseLexer<'input, 'arena, Ext, Input, TF>
where
    Ext: LexerRecog<'input, 'arena, TF, Self> + 'static,
    Input: CharStream<'input>,
    TF: TokenFactory<'input, 'arena> + 'arena,
{
    type Target = Ext;
    fn deref(&self) -> &Self::Target {
        &self.recog
    }
}

impl<'input, 'arena, Ext, Input, TF> DerefMut for BaseLexer<'input, 'arena, Ext, Input, TF>
where
    Ext: LexerRecog<'input, 'arena, TF, Self> + 'static,
    Input: CharStream<'input>,
    TF: TokenFactory<'input, 'arena> + 'arena,
{
    fn deref_mut(&mut self) -> &mut Self::Target {
        &mut self.recog
    }
}

impl<'input, 'arena, Ext, Input, TF> Recognizer<'input, 'arena>
    for BaseLexer<'input, 'arena, Ext, Input, TF>
where
    'input: 'arena,
    Ext: LexerRecog<'input, 'arena, TF, Self> + 'static,
    Input: CharStream<'input>,
    TF: TokenFactory<'input, 'arena> + 'arena,
{
    type Node = EmptyRuleNode<'input, 'arena>;

    fn sempred(
        &mut self,
        localctx: Option<&'arena EmptyRuleNode<'input, 'arena>>,
        rule_index: i32,
        action_index: i32,
    ) -> bool {
        Ext::sempred(localctx, rule_index, action_index, self)
    }

    fn action(
        &mut self,
        localctx: Option<&'arena EmptyRuleNode<'input, 'arena>>,
        rule_index: i32,
        action_index: i32,
    ) {
        Ext::action(localctx, rule_index, action_index, self)
    }
}

/// Default lexer mode id
pub const LEXER_DEFAULT_MODE: usize = 0;
/// Special token type to indicate that lexer should continue current token on next iteration
/// see `Lexer::more()`
pub const LEXER_MORE: i32 = -2;
/// Special token type to indicate that lexer should not return current token
/// usually used to skip whitespaces and comments
/// see `Lexer::skip()`
pub const LEXER_SKIP: i32 = -3;

#[doc(inline)]
pub use super::token::TOKEN_DEFAULT_CHANNEL as LEXER_DEFAULT_TOKEN_CHANNEL;

#[doc(inline)]
pub use super::token::TOKEN_HIDDEN_CHANNEL as LEXER_HIDDEN;

pub(crate) const LEXER_MIN_CHAR_VALUE: i32 = 0x0000;
pub(crate) const LEXER_MAX_CHAR_VALUE: i32 = 0x10FFFF;

impl<'input, 'arena, Ext, Input, TF> BaseLexer<'input, 'arena, Ext, Input, TF>
where
    'input: 'arena,
    Ext: LexerRecog<'input, 'arena, TF, Self> + 'static,
    Input: CharStream<'input>,
    TF: TokenFactory<'input, 'arena> + 'arena,
{
    fn emit_token(&mut self, token: &'arena mut TF::Tok) {
        self.token = Some(token);
    }

    fn emit(&mut self) {
        Ext::before_emit(self);
        let stop = self.get_char_index() - 1;
        let token = self.factory.create(
            Some(self.input.as_mut().unwrap()),
            self.token_type,
            self.text.take(),
            self.channel,
            self.token_start_char_index,
            stop,
            self.token_start_line,
            self.token_start_column,
        );
        self.emit_token(token);
    }

    fn emit_eof(&mut self) {
        let token = self.factory.create(
            None::<&mut Input>,
            super::int_stream::EOF,
            None,
            LEXER_DEFAULT_TOKEN_CHANNEL,
            self.get_char_index(),
            self.get_char_index() - 1,
            self.get_line(),
            self.get_char_position_in_line(),
        );
        self.emit_token(token)
    }

    /// Current position in input stream
    pub fn get_char_index(&self) -> isize {
        self.input.as_ref().unwrap().index()
    }

    /// Current token text
    pub fn get_text<'a>(&'a self) -> Cow<'a, str>
    where
        'input: 'a,
    {
        self.text
            .as_ref()
            .map(|it| Borrowed(it.borrow()))
            // .unwrap_or("")
            .unwrap_or_else(|| {
                self.input
                    .as_ref()
                    .unwrap()
                    .get_text(self.token_start_char_index, self.get_char_index() - 1)
            })
    }

    /// Used from lexer actions to override text of the token that will be emitted next
    pub fn set_text(&mut self, _text: impl Into<String>) {
        self.text = Some(_text.into());
    }

    // fn get_all_tokens(&mut self) -> Vec<TF::Tok> { unimplemented!() }

    // fn get_char_error_display(&self, _c: char) -> String { unimplemented!() }

    /// Add error listener
    pub fn add_error_listener(&mut self, listener: Box<dyn ErrorListener<'input, 'arena, Self>>) {
        self.error_listeners.push(listener);
    }

    /// Remove and drop all error listeners
    pub fn remove_error_listeners(&mut self) {
        self.error_listeners.clear();
    }

    /// Creates new lexer instance
    pub fn new_base_lexer(input: Input, recog: Ext, factory: TF) -> Self {
        let mut lexer = Self {
            interpreter: Some(Box::new(recog.get_atn_simulator())),
            input: Some(input),
            recog,
            factory,
            error_listeners: vec![Box::new(ConsoleErrorListener {})],
            token_start_char_index: 0,
            token_start_line: 0,
            token_start_column: 0,
            current_pos: Rc::new(LexerPosition {
                line: Cell::new(1),
                char_position_in_line: Cell::new(0),
            }),
            token_type: super::token::TOKEN_INVALID_TYPE,
            text: None,
            token: None,
            hit_eof: false,
            channel: super::token::TOKEN_DEFAULT_CHANNEL,
            //            token_factory_source_pair: None,
            mode_stack: Vec::new(),
            mode: self::LEXER_DEFAULT_MODE,
        };
        let pos = lexer.current_pos.clone();
        lexer.interpreter.as_mut().unwrap().current_pos = pos;
        lexer
    }
}

impl<'input, 'arena, L, Input, TF> TokenSource<'input, 'arena, TF>
    for BaseLexer<'input, 'arena, L, Input, TF>
where
    'input: 'arena,
    L: LexerRecog<'input, 'arena, TF, Self> + 'static,
    Input: CharStream<'input>,
    TF: TokenFactory<'input, 'arena> + 'arena,
{
    #[inline]
    #[allow(unused_labels)]
    fn next_token(&mut self) -> &'arena mut TF::Tok {
        assert!(self.input.is_some());

        let _marker = self.input().mark();
        'outer: loop {
            if self.hit_eof {
                self.emit_eof();
                break;
            }
            self.token = None;
            self.channel = LEXER_DEFAULT_TOKEN_CHANNEL;
            self.token_start_column = self
                .interpreter
                .as_ref()
                .unwrap()
                .get_char_position_in_line();
            self.token_start_line = self.interpreter.as_ref().unwrap().get_line();
            self.text = None;
            let index = self.input().index();
            self.token_start_char_index = index;

            'inner: loop {
                self.token_type = TOKEN_INVALID_TYPE;
                // detach from self, to allow self to be passed deeper
                let mut interpreter = self.interpreter.take().unwrap();
                //                    let mut input = self.input.take().unwrap();
                let result = interpreter.match_token(self.mode, self);
                self.interpreter = Some(interpreter);

                let ttype = result.unwrap_or_else(|err| {
                    //                            println!("error, recovering");
                    notify_listeners(self, &err);
                    self.interpreter
                        .as_mut()
                        .unwrap()
                        .recover(err, self.input.as_mut().unwrap());
                    LEXER_SKIP
                });
                //                    self.input = Some(input)

                if self.input().la(1) == super::int_stream::EOF {
                    self.hit_eof = true;
                }

                if self.token_type == TOKEN_INVALID_TYPE {
                    self.token_type = ttype;
                }

                if self.token_type == LEXER_SKIP {
                    continue 'outer;
                }

                if self.token_type != LEXER_MORE {
                    break;
                }
            }

            if self.token.is_none() {
                self.emit();
                break;
            }
        }
        self.input().release(_marker);
        self.token.take().unwrap()
    }

    fn get_line(&self) -> u32 {
        self.current_pos.line.get()
    }

    fn get_char_position_in_line(&self) -> i32 {
        self.current_pos.char_position_in_line.get()
    }

    fn get_input_stream(&mut self) -> Option<&mut dyn IntStream> {
        self.input.as_mut().map(|x| x as _)
    }

    fn get_source_name(&self) -> String {
        self.input
            .as_ref()
            .map(|it| it.get_source_name())
            .unwrap_or("<none>".to_string())
    }

    //    fn set_token_factory<'c: 'b>(&mut self, f: &'c TokenFactory) {
    //        self.factory = f;
    //    }

    fn get_token_factory(&self) -> &TF {
        &self.factory
    }

    fn get_dfa_string(&self) -> String {
        self.get_interpreter()
            .unwrap()
            .get_dfa_for_mode(LEXER_DEFAULT_MODE)
            .to_lexer_string()
    }
}

#[cold]
#[inline(never)]
fn notify_listeners<'input, 'arena, L, Input, TF>(
    lexer: &mut BaseLexer<'input, 'arena, L, Input, TF>,
    e: &ANTLRError,
) where
    'input: 'arena,
    L: LexerRecog<'input, 'arena, TF, BaseLexer<'input, 'arena, L, Input, TF>> + 'static,
    Input: CharStream<'input>,
    TF: TokenFactory<'input, 'arena> + 'arena,
{
    let inner = lexer
        .input
        .as_ref()
        .unwrap()
        .get_text(lexer.token_start_char_index, lexer.get_char_index());
    let text = format!("token recognition error at: '{}'", inner);
    for listener in lexer.error_listeners.iter() {
        listener.syntax_error(
            lexer,
            None,
            lexer.token_start_line,
            lexer.token_start_column,
            &text,
            Some(e),
        )
    }
}

impl<'input, 'arena, L, Input, TF> Lexer<'input, 'arena, Input, TF>
    for BaseLexer<'input, 'arena, L, Input, TF>
where
    'input: 'arena,
    L: LexerRecog<'input, 'arena, TF, Self> + 'static,
    Input: CharStream<'input>,
    TF: TokenFactory<'input, 'arena> + 'arena,
{
    fn input(&mut self) -> &mut Input {
        self.input.as_mut().unwrap()
    }

    fn set_channel(&mut self, v: i32) {
        self.channel = v;
    }

    fn push_mode(&mut self, m: usize) {
        self.mode_stack.push(self.mode);
        self.mode = m;
    }

    fn pop_mode(&mut self) -> Option<usize> {
        self.mode_stack.pop().inspect(|&mode| {
            self.mode = mode;
        })
    }

    fn set_type(&mut self, t: i32) {
        self.token_type = t;
    }

    fn set_mode(&mut self, m: usize) {
        self.mode = m;
    }

    fn more(&mut self) {
        self.set_type(LEXER_MORE)
    }

    fn skip(&mut self) {
        self.set_type(LEXER_SKIP)
    }

    fn reset(&mut self) {
        unimplemented!()
    }

    fn get_interpreter(&self) -> Option<&LexerATNSimulator> {
        self.interpreter.as_deref()
    }
}