oak_tex/lexer/

token_type.rs

use oak_core::{Token, TokenType, UniversalTokenRole};
#[cfg(feature = "serde")]
use serde::{Deserialize, Serialize};

pub type TexToken = Token<TexTokenType>;

#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
#[repr(u8)]
pub enum TexTokenType {
    // Node types
    Root,
    SourceFile,
    Document,

    // TeX commands and environments
    Command,
    Environment,
    BeginEnvironment,
    EndEnvironment,

    // TeX special structures
    MathMode,
    InlineMath,
    DisplayMath,
    Group,
    Superscript,
    Subscript,

    // Arguments and options
    Argument,
    OptionalArgument,
    MandatoryArgument,

    // Text and content
    Text,
    Paragraph,
    Section,
    Subsection,
    Subsubsection,

    // Lists and tables
    List,
    Item,
    Table,
    Row,
    Cell,

    // References and labels
    Label,
    Reference,
    Citation,

    // Figures and floats
    Figure,
    Caption,

    // Error nodes
    Error,

    // TeX keywords and commands
    DocumentClass,
    UsePackage,
    Begin,
    End,
    Section_,
    Subsection_,
    Subsubsection_,
    Chapter,
    Part,
    Title,
    Author,
    Date,
    MakeTitle,
    TableOfContents,
    NewPage,
    ClearPage,

    // Mathematical commands
    Frac,
    Sqrt,
    Sum,
    Int,
    Lim,
    Alpha,
    Beta,
    Gamma,
    Delta,
    Epsilon,
    Zeta,
    Eta,
    Theta,
    Iota,
    Kappa,
    Lambda,
    Mu,
    Nu,
    Xi,
    Omicron,
    Pi,
    Rho,
    Sigma,
    Tau,
    Upsilon,
    Phi,
    Chi,
    Psi,
    Omega,
    VarEpsilon,
    VarTheta,
    VarKappa,
    VarPi,
    VarRho,
    VarSigma,
    VarPhi,
    UpperGamma,
    UpperDelta,
    UpperTheta,
    UpperLambda,
    UpperXi,
    UpperPi,
    UpperSigma,
    UpperUpsilon,
    UpperPhi,
    UpperPsi,
    UpperOmega,

    // Formatting commands
    TextBf,
    TextIt,
    TextSc,
    TextTt,
    Emph,
    Underline,

    // Identifiers and literals
    Identifier,
    StringLiteral,
    Number,

    // Operators and punctuation
    Backslash,
    LeftBrace,
    RightBrace,
    LeftBracket,
    RightBracket,
    LeftParen,
    RightParen,
    Dollar,
    DoubleDollar,
    Ampersand,
    Percent,
    Hash,
    Caret,
    Underscore,
    Tilde,

    // Special characters
    Equal,
    Equals,
    Plus,
    Minus,
    Star,
    Slash,
    Pipe,
    Less,
    LessThan,
    Greater,
    GreaterThan,
    Exclamation,
    Question,
    At,
    Colon,
    Semicolon,
    Comma,
    Dot,

    // Whitespace and comments
    Comment,
    Whitespace,
    Newline,

    // Keywords
    BeginKeyword,
    EndKeyword,
    DocumentclassKeyword,
    UsepackageKeyword,
    SectionKeyword,
    SubsectionKeyword,
    SubsubsectionKeyword,
    ChapterKeyword,
    PartKeyword,
    TitleKeyword,
    AuthorKeyword,
    DateKeyword,
    MaketitleKeyword,
    TableofcontentsKeyword,
    ItemKeyword,
    LabelKeyword,
    RefKeyword,
    CiteKeyword,
    IncludegraphicsKeyword,
    TextbfKeyword,
    TextitKeyword,
    EmphKeyword,

    // End of file
    Eof,
}

impl From<crate::parser::element_type::TexElementType> for TexTokenType {
    fn from(element: crate::parser::element_type::TexElementType) -> Self {
        use crate::parser::element_type::TexElementType as E;
        match element {
            E::Root => Self::Root,
            E::SourceFile => Self::SourceFile,
            E::Document => Self::Document,
            E::Command => Self::Command,
            E::Environment => Self::Environment,
            E::BeginEnvironment => Self::BeginEnvironment,
            E::EndEnvironment => Self::EndEnvironment,
            E::MathMode => Self::MathMode,
            E::InlineMath => Self::InlineMath,
            E::DisplayMath => Self::DisplayMath,
            E::Group => Self::Group,
            E::Superscript => Self::Superscript,
            E::Subscript => Self::Subscript,
            E::Argument => Self::Argument,
            E::OptionalArgument => Self::OptionalArgument,
            E::MandatoryArgument => Self::MandatoryArgument,
            E::Text => Self::Text,
            E::Paragraph => Self::Paragraph,
            E::Section => Self::Section,
            E::Subsection => Self::Subsection,
            E::Subsubsection => Self::Subsubsection,
            E::List => Self::List,
            E::Item => Self::Item,
            E::Table => Self::Table,
            E::Row => Self::Row,
            E::Cell => Self::Cell,
            E::Label => Self::Label,
            E::Reference => Self::Reference,
            E::Citation => Self::Citation,
            E::Figure => Self::Figure,
            E::Caption => Self::Caption,
            E::Error => Self::Error,
            E::DocumentClass => Self::DocumentClass,
            E::UsePackage => Self::UsePackage,
            E::Begin => Self::Begin,
            E::End => Self::End,
            E::Section_ => Self::Section_,
            E::Subsection_ => Self::Subsection_,
            E::Subsubsection_ => Self::Subsubsection_,
            E::Chapter => Self::Chapter,
            E::Part => Self::Part,
            E::Title => Self::Title,
            E::Author => Self::Author,
            E::Date => Self::Date,
            E::MakeTitle => Self::MakeTitle,
            E::TableOfContents => Self::TableOfContents,
            E::NewPage => Self::NewPage,
            E::ClearPage => Self::ClearPage,
            E::Frac => Self::Frac,
            E::Sqrt => Self::Sqrt,
            E::Sum => Self::Sum,
            E::Int => Self::Int,
            E::Lim => Self::Lim,
            E::Alpha => Self::Alpha,
            E::Beta => Self::Beta,
            E::Gamma => Self::Gamma,
            E::Delta => Self::Delta,
            E::Epsilon => Self::Epsilon,
            E::Zeta => Self::Zeta,
            E::Eta => Self::Eta,
            E::Theta => Self::Theta,
            E::Iota => Self::Iota,
            E::Kappa => Self::Kappa,
            E::Lambda => Self::Lambda,
            E::Mu => Self::Mu,
            E::Nu => Self::Nu,
            E::Xi => Self::Xi,
            E::Omicron => Self::Omicron,
            E::Pi => Self::Pi,
            E::Rho => Self::Rho,
            E::Sigma => Self::Sigma,
            E::Tau => Self::Tau,
            E::Upsilon => Self::Upsilon,
            E::Phi => Self::Phi,
            E::Chi => Self::Chi,
            E::Psi => Self::Psi,
            E::Omega => Self::Omega,
            E::VarEpsilon => Self::VarEpsilon,
            E::VarTheta => Self::VarTheta,
            E::VarKappa => Self::VarKappa,
            E::VarPi => Self::VarPi,
            E::VarRho => Self::VarRho,
            E::VarSigma => Self::VarSigma,
            E::VarPhi => Self::VarPhi,
            E::UpperGamma => Self::UpperGamma,
            E::UpperDelta => Self::UpperDelta,
            E::UpperTheta => Self::UpperTheta,
            E::UpperLambda => Self::UpperLambda,
            E::UpperXi => Self::UpperXi,
            E::UpperPi => Self::UpperPi,
            E::UpperSigma => Self::UpperSigma,
            E::UpperUpsilon => Self::UpperUpsilon,
            E::UpperPhi => Self::UpperPhi,
            E::UpperPsi => Self::UpperPsi,
            E::UpperOmega => Self::UpperOmega,
            E::TextBf => Self::TextBf,
            E::TextIt => Self::TextIt,
            E::TextSc => Self::TextSc,
            E::TextTt => Self::TextTt,
            E::Emph => Self::Emph,
            E::Underline => Self::Underline,
            E::Identifier => Self::Identifier,
            E::StringLiteral => Self::StringLiteral,
            E::Number => Self::Number,
            E::Backslash => Self::Backslash,
            E::LeftBrace => Self::LeftBrace,
            E::RightBrace => Self::RightBrace,
            E::LeftBracket => Self::LeftBracket,
            E::RightBracket => Self::RightBracket,
            E::LeftParen => Self::LeftParen,
            E::RightParen => Self::RightParen,
            E::Dollar => Self::Dollar,
            E::DoubleDollar => Self::DoubleDollar,
            E::Ampersand => Self::Ampersand,
            E::Percent => Self::Percent,
            E::Hash => Self::Hash,
            E::Caret => Self::Caret,
            E::Underscore => Self::Underscore,
            E::Tilde => Self::Tilde,
            E::Equal => Self::Equal,
            E::Equals => Self::Equals,
            E::Plus => Self::Plus,
            E::Minus => Self::Minus,
            E::Star => Self::Star,
            E::Slash => Self::Slash,
            E::Pipe => Self::Pipe,
            E::Less => Self::Less,
            E::LessThan => Self::LessThan,
            E::Greater => Self::Greater,
            E::GreaterThan => Self::GreaterThan,
            E::Exclamation => Self::Exclamation,
            E::Question => Self::Question,
            E::At => Self::At,
            E::Colon => Self::Colon,
            E::Semicolon => Self::Semicolon,
            E::Comma => Self::Comma,
            E::Dot => Self::Dot,
            E::Comment => Self::Comment,
            E::Whitespace => Self::Whitespace,
            E::Newline => Self::Newline,
            E::BeginKeyword => Self::BeginKeyword,
            E::EndKeyword => Self::EndKeyword,
            E::DocumentclassKeyword => Self::DocumentclassKeyword,
            E::UsepackageKeyword => Self::UsepackageKeyword,
            E::SectionKeyword => Self::SectionKeyword,
            E::SubsectionKeyword => Self::SubsectionKeyword,
            E::SubsubsectionKeyword => Self::SubsubsectionKeyword,
            E::ChapterKeyword => Self::ChapterKeyword,
            E::PartKeyword => Self::PartKeyword,
            E::TitleKeyword => Self::TitleKeyword,
            E::AuthorKeyword => Self::AuthorKeyword,
            E::DateKeyword => Self::DateKeyword,
            E::MaketitleKeyword => Self::MaketitleKeyword,
            E::TableofcontentsKeyword => Self::TableofcontentsKeyword,
            E::ItemKeyword => Self::ItemKeyword,
            E::LabelKeyword => Self::LabelKeyword,
            E::RefKeyword => Self::RefKeyword,
            E::CiteKeyword => Self::CiteKeyword,
            E::IncludegraphicsKeyword => Self::IncludegraphicsKeyword,
            E::TextbfKeyword => Self::TextbfKeyword,
            E::TextitKeyword => Self::TextitKeyword,
            E::EmphKeyword => Self::EmphKeyword,
            E::Eof => Self::Eof,
        }
    }
}

impl TokenType for TexTokenType {
    type Role = UniversalTokenRole;
    const END_OF_STREAM: Self = Self::Eof;

    fn is_ignored(&self) -> bool {
        matches!(self, Self::Whitespace | Self::Newline | Self::Comment)
    }

    fn role(&self) -> Self::Role {
        match self {
            Self::Whitespace | Self::Newline => UniversalTokenRole::Whitespace,
            Self::Comment => UniversalTokenRole::Comment,
            Self::Eof => UniversalTokenRole::Eof,
            Self::Error => UniversalTokenRole::Error,
            _ => UniversalTokenRole::None,
        }
    }
}