use crate::{FormatToken, Span, TerminalIndex, ToSpan};
use std::borrow::Cow;
use std::convert::From;
use std::fmt::{Debug, Display, Error, Formatter};
use super::{Location, TokenNumber};
pub const EOI: TerminalIndex = 0;
pub const NEW_LINE: TerminalIndex = 1;
pub const WHITESPACE: TerminalIndex = 2;
pub const LINE_COMMENT: TerminalIndex = 3;
pub const BLOCK_COMMENT: TerminalIndex = 4;
pub const FIRST_USER_TOKEN: TerminalIndex = 5;
const EOI_TOKEN: &str = "$";
#[derive(Debug, Clone, Default, Eq, PartialEq)]
pub struct Token<'t> {
pub(crate) text: Cow<'t, str>,
pub token_type: TerminalIndex,
pub location: Location,
pub token_number: TokenNumber,
}
impl<'t> Token<'t> {
pub fn eoi(token_number: TokenNumber) -> Self {
Self {
text: EOI_TOKEN.into(),
token_type: EOI,
location: Location::default(),
token_number,
}
}
pub fn with<T>(
text: T,
token_type: TerminalIndex,
location: Location,
token_number: TokenNumber,
) -> Self
where
T: Into<Cow<'t, str>>,
{
Self {
text: text.into(),
token_type,
location,
token_number,
}
}
pub fn with_location(mut self, location: Location) -> Self {
self.location = location;
self
}
pub fn with_type(mut self, token_type: TerminalIndex) -> Self {
self.token_type = token_type;
self
}
#[inline]
pub fn is_skip_token(&self) -> bool {
self.token_type > EOI && self.token_type < FIRST_USER_TOKEN
}
#[inline]
pub fn is_comment_token(&self) -> bool {
self.token_type == LINE_COMMENT || self.token_type == BLOCK_COMMENT
}
pub fn text(&self) -> &str {
self.text.as_ref()
}
pub fn to_owned(&self) -> Token<'static> {
Token {
text: Cow::Owned(self.text.clone().into_owned()),
token_type: self.token_type,
location: self.location.clone(),
token_number: self.token_number,
}
}
pub fn into_owned(self) -> Token<'static> {
Token {
text: Cow::Owned(self.text.into_owned()),
token_type: self.token_type,
location: self.location,
token_number: self.token_number,
}
}
}
impl Display for Token<'_> {
fn fmt(&self, f: &mut Formatter<'_>) -> std::result::Result<(), Error> {
let (c1, c2) = if self.text.starts_with('\'') {
('<', '>')
} else {
('\'', '\'')
};
write!(
f,
"{}{}{}, Ty:{}, at {}",
c1, self.text, c2, self.token_type, self.location
)
}
}
impl FormatToken for Token<'_> {
fn format(&self, terminal_names: &'static [&'static str]) -> String {
let name = terminal_names[self.token_type as usize];
format!(
"{} ({}) at {}[{}]",
self.text, name, self.location, self.token_number,
)
}
}
impl From<&Token<'_>> for std::ops::Range<usize> {
fn from(token: &Token<'_>) -> Self {
(&token.location).into()
}
}
impl From<&Token<'_>> for Location {
fn from(token: &Token<'_>) -> Self {
token.location.clone()
}
}
impl From<&Token<'_>> for Span {
fn from(token: &Token<'_>) -> Self {
(Into::<std::ops::Range<usize>>::into(&token.location)).into()
}
}
impl ToSpan for Token<'_> {
fn span(&self) -> Span {
self.into()
}
}