dbt-antlr4 1.3.6

//! Symbols that parser works on
use std::fmt::Formatter;
use std::fmt::{Debug, Display};
use std::sync::LazyLock;

/// Type of tokens that parser considers invalid
pub const TOKEN_INVALID_TYPE: i32 = 0;
/// Type of tokens that DFA can use to advance to next state without consuming actual input token.
/// Should not be created by downstream implementations.
pub const TOKEN_EPSILON: i32 = -2;
/// Min token type that can be assigned to tokens created by downstream implementations.
pub const TOKEN_MIN_USER_TOKEN_TYPE: i32 = 1;
/// Type of EOF token
pub const TOKEN_EOF: i32 = crate::int_stream::EOF;
/// Default channel lexer emits tokens to
pub const TOKEN_DEFAULT_CHANNEL: i32 = 0;
/// Predefined additional channel for lexer to assign tokens to
pub const TOKEN_HIDDEN_CHANNEL: i32 = 1;
/// Shorthand for TOKEN_HIDDEN_CHANNEL
pub const HIDDEN: i32 = TOKEN_HIDDEN_CHANNEL;

/// Implemented by tokens that are produced by a `TokenFactory`
#[allow(missing_docs)]
pub trait Token: Debug + Display {
    /***
     * Begin `Token` interface
     */

    /// Get the text of the token
    fn get_text(&self) -> &str;

    /// Get the token type of the token
    fn get_token_type(&self) -> i32;

    /// The line number on which the first character of this token was
    /// matched. The first line in the input is line 1.
    fn get_line(&self) -> u32;

    /// The index of the first character of this token relative to the beginning
    /// of the line at which it occurs. The first character on a line has
    /// position 0.
    fn get_char_position_in_line(&self) -> i32;

    /// Returns the channel number this token was assigned to. Each token can
    /// arrive on a different channel, but the parser only "tunes" to a single
    /// channel. The parser ignores everything not on DEFAULT_CHANNEL.
    fn get_channel(&self) -> i32;

    /// An index from 0..n-1 of the token object in the token stream.
    /// This must be valid in order to print token streams and use
    /// token stream rewinding.
    ///
    /// Return -1 to indicate that the token was conjured up and does not
    /// have a valid index.
    fn get_token_index(&self) -> isize;

    /// The starting character index of the token
    ///
    /// This method is optional; return -1 if not implemented.
    fn get_start_index(&self) -> isize;

    /// The last character index of the token
    ///
    /// This method is optional; return -1 if not implemented.
    fn get_stop_index(&self) -> isize;

    // fn get_token_source(&self) -> &dyn TokenSource;
    // fn get_input_stream(&self) -> &dyn CharStream;

    /***
     * Begin `WritableToken` interface
     */

    fn set_text(&mut self, _text: String);

    fn set_type(&mut self, _ttype: i32);

    fn set_line(&mut self, _line: u32);

    fn set_char_position_in_line(&mut self, _pos: i32);

    fn set_channel(&mut self, _channel: i32);

    fn set_token_index(&mut self, _v: isize);
}

/// Token that owns its data
pub type OwningToken = TokenImpl<'static, String>;

/// Most versatile Token that uses Cow to save data
/// Can be used seamlessly switch from owned to zero-copy parsing
pub type CommonToken<'input> = TokenImpl<'input, &'input str>;

pub trait TextType<'input>: AsRef<str> + Debug + Display {
    fn set_text(&mut self, _text: String);
}

impl<'input> TextType<'input> for &'input str {
    fn set_text(&mut self, _text: String) {
        panic!("Cannot set text of &str, it is immutable");
    }
}

impl<'input> TextType<'input> for String {
    fn set_text(&mut self, text: String) {
        *self = text;
    }
}

const TOKEN_TYPE_BITS: u32 = 22;
const TOKEN_CHANNEL_BITS: u32 = 32 - TOKEN_TYPE_BITS;
const TOKEN_TYPE_MASK: i32 = (1 << TOKEN_TYPE_BITS) - 1;
const TOKEN_CHANNEL_MASK: i32 = !TOKEN_TYPE_MASK;
const TOKEN_TYPE_MAX: i32 = (TOKEN_TYPE_MASK as u32 >> 1) as i32;
const TOKEN_TYPE_MIN: i32 = -TOKEN_TYPE_MAX;
const TOKEN_CHANNEL_MAX: i32 = 1i32 << (TOKEN_CHANNEL_BITS - 1);
const TOKEN_CHANNEL_MIN: i32 = -TOKEN_CHANNEL_MAX;

#[derive(Clone, Debug)]
#[allow(missing_docs)]
pub struct TokenImpl<'input, T: TextType<'input>> {
    //    source: Option<(Box<TokenSource>,Box<CharStream>)>,
    type_and_channel: i32,
    pub start: i32,
    pub stop: i32,
    pub token_index: i32,
    pub line: u32,
    pub column: i32,
    pub text: T,

    _input: std::marker::PhantomData<&'input str>,
}

impl<'input, T: TextType<'input>> TokenImpl<'input, T> {
    #[allow(clippy::too_many_arguments)]
    pub fn new(
        token_type: i32,
        channel: i32,
        start: i32,
        stop: i32,
        token_index: i32,
        line: u32,
        column: i32,
        text: T,
    ) -> Self {
        Self {
            type_and_channel: (channel << TOKEN_TYPE_BITS) | (token_type & TOKEN_TYPE_MASK),
            start,
            stop,
            token_index,
            line,
            column,
            text,

            _input: std::marker::PhantomData,
        }
    }
}

impl<'input, T: TextType<'input>> Display for TokenImpl<'input, T> {
    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
        let txt = self.get_text();
        let txt = txt.replace("\n", "\\n");
        let txt = txt.replace("\r", "\\r");
        let txt = txt.replace("\t", "\\t");
        //        let txt = escape_whitespaces(txt,false);
        f.write_fmt(format_args!(
            "[@{},{}:{}='{}',<{}>{},{}:{}]",
            self.get_token_index(),
            self.start,
            self.stop,
            txt,
            self.get_token_type(),
            if self.get_channel() > 0 {
                ",channel=".to_string() + self.get_channel().to_string().as_str()
            } else {
                String::new()
            },
            self.line,
            self.column
        ))
    }
}

impl<'input, T: TextType<'input>> Token for TokenImpl<'input, T> {
    #[inline(always)]
    fn get_token_type(&self) -> i32 {
        (self.type_and_channel << TOKEN_CHANNEL_BITS) >> TOKEN_CHANNEL_BITS
    }

    #[inline(always)]
    fn get_channel(&self) -> i32 {
        self.type_and_channel >> TOKEN_TYPE_BITS
    }

    #[inline(always)]
    fn get_start_index(&self) -> isize {
        self.start as isize
    }

    #[inline(always)]
    fn get_stop_index(&self) -> isize {
        self.stop as isize
    }

    #[inline(always)]
    fn get_line(&self) -> u32 {
        self.line
    }

    #[inline(always)]
    fn get_char_position_in_line(&self) -> i32 {
        self.column
    }

    // fn get_source(&self) -> Option<(Box<dyn TokenSource>, Box<dyn CharStream>)> {
    //     unimplemented!()
    // }

    #[inline(always)]
    fn get_text(&self) -> &str {
        if self.get_token_type() == TOKEN_EOF {
            "<EOF>"
        } else {
            self.text.as_ref()
        }
    }

    #[inline(always)]
    fn get_token_index(&self) -> isize {
        self.token_index as isize
    }

    fn set_token_index(&mut self, _v: isize) {
        self.token_index = _v as i32;
    }

    fn set_text(&mut self, text: String) {
        TextType::set_text(&mut self.text, text);
    }

    #[inline]
    fn set_type(&mut self, ttype: i32) {
        debug_assert!(
            (TOKEN_TYPE_MIN..=TOKEN_TYPE_MAX).contains(&ttype),
            "Token type {} is out of range ({}..={})",
            ttype,
            TOKEN_TYPE_MIN,
            TOKEN_TYPE_MAX
        );

        self.type_and_channel =
            (self.type_and_channel & !(TOKEN_TYPE_MASK)) | (ttype & TOKEN_TYPE_MASK);
    }

    fn set_line(&mut self, _line: u32) {
        self.line = _line;
    }

    fn set_char_position_in_line(&mut self, _pos: i32) {
        self.column = _pos;
    }

    #[inline]
    fn set_channel(&mut self, channel: i32) {
        debug_assert!(
            (TOKEN_CHANNEL_MIN..=TOKEN_CHANNEL_MAX).contains(&channel),
            "Token channel {} is out of range ({}..={})",
            channel,
            TOKEN_CHANNEL_MIN,
            TOKEN_CHANNEL_MAX
        );

        self.type_and_channel =
            (self.type_and_channel & !(TOKEN_CHANNEL_MASK)) | (channel << TOKEN_TYPE_BITS);
    }
}

impl Default for &'_ CommonToken<'_> {
    fn default() -> Self {
        &INVALID_COMMON
    }
}

impl From<&dyn Token> for OwningToken {
    fn from(value: &dyn Token) -> Self {
        OwningToken::new(
            value.get_token_type(),
            value.get_channel(),
            value.get_start_index() as i32,
            value.get_stop_index() as i32,
            value.get_token_index() as i32,
            value.get_line(),
            value.get_char_position_in_line(),
            value.get_text().to_string(),
        )
    }
}

pub(crate) static INVALID_COMMON: LazyLock<Box<CommonToken<'static>>> = LazyLock::new(|| {
    Box::new(CommonToken::new(
        TOKEN_INVALID_TYPE,
        0,
        -1,
        -1,
        -1,
        0,
        0,
        "<invalid>",
    ))
});