tex_engine 0.0.1

/*!
    [`Token`]s are the basic "characters" a TeX Engine operates on.

    There are two kinds of [`Token`]s:
    - A *Character Token* is a pair of a character (of type `Char:`[`CharType`]) and its [`CategoryCode`],
        represented by [`BaseToken::Char`].
    - A *Control Sequence Token* is a [`TeXStr`] (a string of characters) and is represented by [`BaseToken::CS`],
        usually generated by `\` followed by a string of letters (or a single non-letter character).
 */

use std::fmt::{Debug, Display, Formatter};
use crate::tex::catcodes::CategoryCode;
use crate::tex::commands::Command;
use crate::utils::Ptr;
use crate::utils::strings::{CharType, TeXStr};

/// A [`BaseToken`] bundles the actually TeX-relevant data of a [`Token`], which is cloned often
/// and required by all [`Token`] implementations
#[derive(Clone,PartialEq)]
pub enum BaseToken<C:CharType> {
    /// A control sequence token with the provided name
    CS(TeXStr<C>),
    /// An active character token with the provided character
    Char(C, CategoryCode)
}
impl<C:CharType> Display for BaseToken<C> {
    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
        match self {
            BaseToken::Char(c, _) => write!(f, "{}", c.char_str()),
            BaseToken::CS(n) => write!(f, "\\{}", n)
        }
    }
}
impl<C:CharType> Debug for BaseToken<C> {
    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
        use ansi_term::Colour::*;
        use CategoryCode::*;
        match self {
            BaseToken::Char(c, cc) => write!(f, "'{}'", match cc {
                BeginGroup => Green.bold().paint(c.char_str()),
                EndGroup => Green.paint(c.char_str()),
                Active => Red.paint(c.char_str()),
                Space => ansi_term::Style::new().paint(c.char_str()),
                Parameter => Yellow.paint(c.char_str()),
                AlignmentTab => Blue.paint(c.char_str()),
                MathShift => Purple.paint(c.char_str()),
                Subscript => Cyan.paint(c.char_str()),
                Superscript => Cyan.bold().paint(c.char_str()),
                Letter => White.bold().paint(c.char_str()),
                Comment => White.paint(c.char_str()),
                _ => ansi_term::Style::new().paint(c.char_str())
            }),
            BaseToken::CS(n) => write!(f, "{}{}", Red.paint("\\"), Red.paint(n.to_string()))
        }
    }
}

pub trait Token:PartialEq+Clone+Display+Debug+'static{
    type Char:CharType;
    /// The actual TeX-relevant data of the [`Token`]
    fn base(&self) -> &BaseToken<Self::Char>;
    /// The [`CategoryCode`] of the [`Token`]
    fn catcode(&self) -> CategoryCode {
        match self.base() {
            BaseToken::Char(_, cat) => *cat,
            BaseToken::CS(_) => CategoryCode::Escape
        }
    }
    /// Construct a new [`Token`] from a [`BaseToken`]
    fn new(base:BaseToken<Self::Char>,sourceref:Option<(Ptr<String>,(usize,usize),(usize,usize))>) -> Self;
    /// Clone this [`Token`] with a new [`SourceReference`] of type [`SourceReference::Expansion`]
    /// (if the [`Token`] is a [`TokenWithSourceref`]).
    fn with_ref(&self, token:&Ptr<Self>, cmd:&Ptr<Command<Self>>) -> Self;
    fn sourceref_trace(&self) -> Option<String>;

    /// Parse a string into a list of [`Token`]s of [`CategoryCode::Other`]
    fn from_str(s:String) -> Vec<Self>;
}

impl<C:CharType> Token for BaseToken<C> {
    type Char = C;
    fn base(&self) -> &BaseToken<C> { self }
    fn new(base:BaseToken<C>,sourceref:Option<(Ptr<String>,(usize,usize),(usize,usize))>) -> Self { base }
    fn with_ref(&self, _: &Ptr<Self>, _: &Ptr<Command<Self>>) -> Self {
        self.clone()
    }
    fn sourceref_trace(&self) -> Option<String> { None }
    fn from_str(s:String) -> Vec<Self> {
        let mut ret = Vec::with_capacity(s.len());
        let mut iter = s.as_bytes().to_vec().into_iter();
        while let Some(c) = C::from_u8_iter(&mut iter) {
            ret.push(BaseToken::Char(c, CategoryCode::Other))
        }
        ret
    }
}

/// A list of [`Token`]s
pub struct TokenList<T:Token>(pub Vec<T>);
impl<T:Token> Into<TokenList<T>> for Vec<T> {
    fn into(self) -> TokenList<T> { TokenList(self) }
}
impl<T:Token> Into<Vec<T>> for TokenList<T> {
    fn into(self) -> Vec<T> { self.0 }
}
impl<T:Token> Display for TokenList<T> {
    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
        for t in &self.0 {
            write!(f, "{}", t.base())?;
        }
        Ok(())
    }
}
impl<T:Token> Debug for TokenList<T> {
    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
        for t in &self.0 {
            write!(f, "{:?}", t.base())?;
        }
        Ok(())
    }
}

/// A [`Token`] bundling a [`BaseToken`] with a [`SourceReference`].
#[derive(Clone)]
pub struct TokenWithSourceref<C:CharType>{
    /// The actual TeX-relevant data of the [`Token`]
    pub base: BaseToken<C>,
    /// The [`SourceReference`] of the [`Token`]
    pub sourceref: Option<SourceReference<Self>>
}
impl<C:CharType> Token for TokenWithSourceref<C> {
    type Char = C;
    fn base(&self) -> &BaseToken<C> { &self.base }
    fn new(base:BaseToken<C>,sourceref:Option<(Ptr<String>,(usize,usize),(usize,usize))>) -> Self {
        Self { base, sourceref: sourceref.map(|tr| SourceReference::File {
            file: tr.0,
            start: tr.1,
            end: tr.2
        }) }
    }
    fn with_ref(&self, token:&Ptr<Self>, cmd:&Ptr<Command<Self>>) -> Self {
        Self {
            base: self.base.clone(),
            sourceref: Some(SourceReference::Expansion {token:token.clone(),cmd:cmd.clone()})
        }
    }
    fn sourceref_trace(&self) -> Option<String> {
        match &self.sourceref {
            Some(SourceReference::File {file, start,end}) => Some(format!("File {} at {}:{} - {}:{}", file, start.0,start.1,end.0,end.1)),
            Some(SourceReference::Expansion {token, ..}) => {
                let mut trace = format!("Expanded from {}",token);
                match token.sourceref_trace() {
                    Some(s) => {
                        trace.push_str("\n - ");
                        trace.push_str(&s)
                    },
                    None => ()
                }
                Some(trace)
            },
            None => None
        }
    }
    fn from_str(s: String) -> Vec<Self> {
        BaseToken::from_str(s).into_iter().map(|b| Self { base: b, sourceref: None }).collect()
    }
}
impl<C:CharType> PartialEq for TokenWithSourceref<C> {
    fn eq(&self, other: &Self) -> bool { self.base == other.base }
}
impl<C:CharType> Display for TokenWithSourceref<C> {
    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { Display::fmt(&self.base,f) }
}
impl<C:CharType> Debug for TokenWithSourceref<C> {
    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
        match self.sourceref {
            Some(ref s) => write!(f, "{:?} at {:?}", self.base, s),
            None => write!(f, "{:?}", self.base)
        }
    }
}


/// A [`SourceReference`] allows tracking the origin of a [`Token`]. It is either a file reference
/// or a reference to an expansion: The result of expanding a control sequence token.
#[derive(Clone)]
pub enum SourceReference<T:Token> {
    /// A reference to a file, with the file name and the start and end position of the
    /// [`Token`] in the file.
    File{file: Ptr<String>,start:(usize,usize),end:(usize,usize)},
    /// A reference to an expansion, with the [`Token`] that was expanded via [`Command`].
    Expansion{token: Ptr<T>,cmd:Ptr<Command<T>>}
}
impl<T:Token> Debug for SourceReference<T> {
    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
        match self {
            SourceReference::File{file,start,end} => write!(f, "File {}; {:?} - {:?}", file, start, end),
            SourceReference::Expansion{token,cmd} => write!(f, "Expansion of {} via {:?}", token, cmd)
        }
    }
}