oak_tex/lexer/
token_type.rs

1use oak_core::{Token, TokenType, UniversalTokenRole};
2
3/// A token in a TeX source file.
4pub type TexToken = Token<TexTokenType>;
5
6/// TeX token types.
7#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
8#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
9#[repr(u8)]
10pub enum TexTokenType {
11    /// Root node of the TeX file.
12    Root,
13    /// Source file.
14    SourceFile,
15    /// Document environment.
16    Document,
17
18    /// Generic command.
19    Command,
20    /// Generic environment.
21    Environment,
22    /// Start of an environment (\begin).
23    BeginEnvironment,
24    /// End of an environment (\end).
25    EndEnvironment,
26
27    /// Math mode content.
28    MathMode,
29    /// Inline math mode ($...$ or \(...\)).
30    InlineMath,
31    /// Display math mode ($$...$$ or \[...\]).
32    DisplayMath,
33    /// Grouped content ({...}).
34    Group,
35    /// Superscript (^).
36    Superscript,
37    /// Subscript (_).
38    Subscript,
39
40    /// Generic argument.
41    Argument,
42    /// Optional argument ([...]).
43    OptionalArgument,
44    /// Mandatory argument ({...}).
45    MandatoryArgument,
46
47    /// Text content.
48    Text,
49    /// Paragraph.
50    Paragraph,
51    /// Section.
52    Section,
53    /// Subsection.
54    Subsection,
55    /// Subsubsection.
56    Subsubsection,
57
58    /// List environment.
59    List,
60    /// Item in a list (\item).
61    Item,
62    /// Table environment.
63    Table,
64    /// Row in a table.
65    Row,
66    /// Cell in a table.
67    Cell,
68
69    /// Label definition (\label).
70    Label,
71    /// Reference to a label (\ref).
72    Reference,
73    /// Citation (\cite).
74    Citation,
75
76    /// Figure environment.
77    Figure,
78    /// Caption (\caption).
79    Caption,
80
81    /// Error token.
82    Error,
83
84    /// \documentclass command.
85    DocumentClass,
86    /// \usepackage command.
87    UsePackage,
88    /// \begin command.
89    Begin,
90    /// \end command.
91    End,
92    /// \section command.
93    Section_,
94    /// \subsection command.
95    Subsection_,
96    /// \subsubsection command.
97    Subsubsection_,
98    /// \chapter command.
99    Chapter,
100    /// \part command.
101    Part,
102    /// \title command.
103    Title,
104    /// \author command.
105    Author,
106    /// \date command.
107    Date,
108    /// \maketitle command.
109    MakeTitle,
110    /// \tableofcontents command.
111    TableOfContents,
112    /// \newpage command.
113    NewPage,
114    /// \clearpage command.
115    ClearPage,
116
117    /// \frac command.
118    Frac,
119    /// \sqrt command.
120    Sqrt,
121    /// \sum command.
122    Sum,
123    /// \int command.
124    Int,
125    /// \lim command.
126    Lim,
127    /// \alpha Greek letter.
128    Alpha,
129    /// \beta Greek letter.
130    Beta,
131    /// \gamma Greek letter.
132    Gamma,
133    /// \delta Greek letter.
134    Delta,
135    /// \epsilon Greek letter.
136    Epsilon,
137    /// \zeta Greek letter.
138    Zeta,
139    /// \eta Greek letter.
140    Eta,
141    /// \theta Greek letter.
142    Theta,
143    /// \iota Greek letter.
144    Iota,
145    /// \kappa Greek letter.
146    Kappa,
147    /// \lambda Greek letter.
148    Lambda,
149    /// \mu Greek letter.
150    Mu,
151    /// \nu Greek letter.
152    Nu,
153    /// \xi Greek letter.
154    Xi,
155    /// \omicron Greek letter.
156    Omicron,
157    /// \pi Greek letter.
158    Pi,
159    /// \rho Greek letter.
160    Rho,
161    /// \sigma Greek letter.
162    Sigma,
163    /// \tau Greek letter.
164    Tau,
165    /// \upsilon Greek letter.
166    Upsilon,
167    /// \phi Greek letter.
168    Phi,
169    /// \chi Greek letter.
170    Chi,
171    /// \psi Greek letter.
172    Psi,
173    /// \omega Greek letter.
174    Omega,
175    /// \varepsilon Greek letter.
176    VarEpsilon,
177    /// \vartheta Greek letter.
178    VarTheta,
179    /// \varkappa Greek letter.
180    VarKappa,
181    /// \varpi Greek letter.
182    VarPi,
183    /// \varrho Greek letter.
184    VarRho,
185    /// \varsigma Greek letter.
186    VarSigma,
187    /// \varphi Greek letter.
188    VarPhi,
189    /// \Gamma Greek letter.
190    UpperGamma,
191    /// \Delta Greek letter.
192    UpperDelta,
193    /// \Theta Greek letter.
194    UpperTheta,
195    /// \Lambda Greek letter.
196    UpperLambda,
197    /// \Xi Greek letter.
198    UpperXi,
199    /// \Pi Greek letter.
200    UpperPi,
201    /// \Sigma Greek letter.
202    UpperSigma,
203    /// \Upsilon Greek letter.
204    UpperUpsilon,
205    /// \Phi Greek letter.
206    UpperPhi,
207    /// \Psi Greek letter.
208    UpperPsi,
209    /// \Omega Greek letter.
210    UpperOmega,
211
212    /// \textbf command.
213    TextBf,
214    /// \textit command.
215    TextIt,
216    /// \textsc command.
217    TextSc,
218    /// \texttt command.
219    TextTt,
220    /// \emph command.
221    Emph,
222    /// \underline command.
223    Underline,
224
225    /// Identifier.
226    Identifier,
227    /// String literal.
228    StringLiteral,
229    /// Numeric literal.
230    Number,
231
232    /// Backslash (\).
233    Backslash,
234    /// Left brace ({).
235    LeftBrace,
236    /// Right brace (}).
237    RightBrace,
238    /// Left bracket ([).
239    LeftBracket,
240    /// Right bracket (]).
241    RightBracket,
242    /// Left parenthesis (().
243    LeftParen,
244    /// Right parenthesis ()).
245    RightParen,
246    /// Dollar sign ($).
247    Dollar,
248    /// Double dollar sign ($$).
249    DoubleDollar,
250    /// Ampersand (&).
251    Ampersand,
252    /// Percent sign (%).
253    Percent,
254    /// Hash sign (#).
255    Hash,
256    /// Caret sign (^).
257    Caret,
258    /// Underscore (_).
259    Underscore,
260    /// Tilde (~).
261    Tilde,
262
263    /// Equal sign (=).
264    Equal,
265    /// Equals sign (alternative).
266    Equals,
267    /// Plus sign (+).
268    Plus,
269    /// Minus sign (-).
270    Minus,
271    /// Star sign (*).
272    Star,
273    /// Slash sign (/).
274    Slash,
275    /// Pipe sign (|).
276    Pipe,
277    /// Less than sign (<).
278    Less,
279    /// Less than sign (alternative).
280    LessThan,
281    /// Greater than sign (>).
282    Greater,
283    /// Greater than sign (alternative).
284    GreaterThan,
285    /// Exclamation mark (!).
286    Exclamation,
287    /// Question mark (?).
288    Question,
289    /// At sign (@).
290    At,
291    /// Colon (:).
292    Colon,
293    /// Semicolon (;).
294    Semicolon,
295    /// Comma (,).
296    Comma,
297    /// Dot (.).
298    Dot,
299
300    /// Comment (% ...).
301    Comment,
302    /// Whitespace.
303    Whitespace,
304    /// Newline.
305    Newline,
306
307    /// 'begin' keyword.
308    BeginKeyword,
309    /// 'end' keyword.
310    EndKeyword,
311    /// 'documentclass' keyword.
312    DocumentclassKeyword,
313    /// 'usepackage' keyword.
314    UsepackageKeyword,
315    /// 'section' keyword.
316    SectionKeyword,
317    /// 'subsection' keyword.
318    SubsectionKeyword,
319    /// 'subsubsection' keyword.
320    SubsubsectionKeyword,
321    /// 'chapter' keyword.
322    ChapterKeyword,
323    /// 'part' keyword.
324    PartKeyword,
325    /// 'title' keyword.
326    TitleKeyword,
327    /// 'author' keyword.
328    AuthorKeyword,
329    /// 'date' keyword.
330    DateKeyword,
331    /// 'maketitle' keyword.
332    MaketitleKeyword,
333    /// 'tableofcontents' keyword.
334    TableofcontentsKeyword,
335    /// 'item' keyword.
336    ItemKeyword,
337    /// 'label' keyword.
338    LabelKeyword,
339    /// 'ref' keyword.
340    RefKeyword,
341    /// 'cite' keyword.
342    CiteKeyword,
343    /// 'includegraphics' keyword.
344    IncludegraphicsKeyword,
345    /// 'textbf' keyword.
346    TextbfKeyword,
347    /// 'textit' keyword.
348    TextitKeyword,
349    /// 'emph' keyword.
350    EmphKeyword,
351
352    /// End of stream.
353    Eof,
354}
355
356impl TokenType for TexTokenType {
357    type Role = UniversalTokenRole;
358    const END_OF_STREAM: Self = Self::Eof;
359
360    fn is_ignored(&self) -> bool {
361        matches!(self, Self::Whitespace | Self::Newline | Self::Comment)
362    }
363
364    fn role(&self) -> Self::Role {
365        match self {
366            Self::Whitespace | Self::Newline => UniversalTokenRole::Whitespace,
367            Self::Comment => UniversalTokenRole::Comment,
368            Self::BeginKeyword
369            | Self::EndKeyword
370            | Self::DocumentclassKeyword
371            | Self::UsepackageKeyword
372            | Self::SectionKeyword
373            | Self::SubsectionKeyword
374            | Self::SubsubsectionKeyword
375            | Self::ChapterKeyword
376            | Self::PartKeyword
377            | Self::TitleKeyword
378            | Self::AuthorKeyword
379            | Self::DateKeyword
380            | Self::MaketitleKeyword
381            | Self::TableofcontentsKeyword
382            | Self::ItemKeyword
383            | Self::LabelKeyword
384            | Self::RefKeyword
385            | Self::CiteKeyword
386            | Self::IncludegraphicsKeyword
387            | Self::TextbfKeyword
388            | Self::TextitKeyword
389            | Self::EmphKeyword => UniversalTokenRole::Keyword,
390            Self::Number | Self::StringLiteral => UniversalTokenRole::Literal,
391            Self::Backslash
392            | Self::Plus
393            | Self::Minus
394            | Self::Star
395            | Self::Slash
396            | Self::Pipe
397            | Self::Less
398            | Self::LessThan
399            | Self::Greater
400            | Self::GreaterThan
401            | Self::Equal
402            | Self::Equals
403            | Self::Caret
404            | Self::Underscore
405            | Self::Tilde
406            | Self::Dollar
407            | Self::DoubleDollar
408            | Self::Ampersand
409            | Self::Hash
410            | Self::At => UniversalTokenRole::Operator,
411            Self::LeftBrace | Self::RightBrace | Self::LeftBracket | Self::RightBracket | Self::LeftParen | Self::RightParen | Self::Comma | Self::Dot | Self::Colon | Self::Semicolon | Self::Exclamation | Self::Question | Self::Percent => {
412                UniversalTokenRole::Punctuation
413            }
414            Self::Identifier | Self::Text => UniversalTokenRole::Name,
415            Self::Eof => UniversalTokenRole::Eof,
416            Self::Error => UniversalTokenRole::Error,
417            _ => UniversalTokenRole::None,
418        }
419    }
420}
421
422impl From<crate::parser::element_type::TexElementType> for TexTokenType {
423    fn from(element: crate::parser::element_type::TexElementType) -> Self {
424        use crate::parser::element_type::TexElementType as E;
425        match element {
426            E::Root => Self::Root,
427            E::SourceFile => Self::SourceFile,
428            E::Document => Self::Document,
429            E::Command => Self::Command,
430            E::Environment => Self::Environment,
431            E::BeginEnvironment => Self::BeginEnvironment,
432            E::EndEnvironment => Self::EndEnvironment,
433            E::MathMode => Self::MathMode,
434            E::InlineMath => Self::InlineMath,
435            E::DisplayMath => Self::DisplayMath,
436            E::Group => Self::Group,
437            E::Superscript => Self::Superscript,
438            E::Subscript => Self::Subscript,
439            E::Argument => Self::Argument,
440            E::OptionalArgument => Self::OptionalArgument,
441            E::MandatoryArgument => Self::MandatoryArgument,
442            E::Text => Self::Text,
443            E::Paragraph => Self::Paragraph,
444            E::Section => Self::Section,
445            E::Subsection => Self::Subsection,
446            E::Subsubsection => Self::Subsubsection,
447            E::List => Self::List,
448            E::Item => Self::Item,
449            E::Table => Self::Table,
450            E::Row => Self::Row,
451            E::Cell => Self::Cell,
452            E::Label => Self::Label,
453            E::Reference => Self::Reference,
454            E::Citation => Self::Citation,
455            E::Figure => Self::Figure,
456            E::Caption => Self::Caption,
457            E::Error => Self::Error,
458            E::DocumentClass => Self::DocumentClass,
459            E::UsePackage => Self::UsePackage,
460            E::Begin => Self::Begin,
461            E::End => Self::End,
462            E::Section_ => Self::Section_,
463            E::Subsection_ => Self::Subsection_,
464            E::Subsubsection_ => Self::Subsubsection_,
465            E::Chapter => Self::Chapter,
466            E::Part => Self::Part,
467            E::Title => Self::Title,
468            E::Author => Self::Author,
469            E::Date => Self::Date,
470            E::MakeTitle => Self::MakeTitle,
471            E::TableOfContents => Self::TableOfContents,
472            E::NewPage => Self::NewPage,
473            E::ClearPage => Self::ClearPage,
474            E::Frac => Self::Frac,
475            E::Sqrt => Self::Sqrt,
476            E::Sum => Self::Sum,
477            E::Int => Self::Int,
478            E::Lim => Self::Lim,
479            E::Alpha => Self::Alpha,
480            E::Beta => Self::Beta,
481            E::Gamma => Self::Gamma,
482            E::Delta => Self::Delta,
483            E::Epsilon => Self::Epsilon,
484            E::Zeta => Self::Zeta,
485            E::Eta => Self::Eta,
486            E::Theta => Self::Theta,
487            E::Iota => Self::Iota,
488            E::Kappa => Self::Kappa,
489            E::Lambda => Self::Lambda,
490            E::Mu => Self::Mu,
491            E::Nu => Self::Nu,
492            E::Xi => Self::Xi,
493            E::Omicron => Self::Omicron,
494            E::Pi => Self::Pi,
495            E::Rho => Self::Rho,
496            E::Sigma => Self::Sigma,
497            E::Tau => Self::Tau,
498            E::Upsilon => Self::Upsilon,
499            E::Phi => Self::Phi,
500            E::Chi => Self::Chi,
501            E::Psi => Self::Psi,
502            E::Omega => Self::Omega,
503            E::VarEpsilon => Self::VarEpsilon,
504            E::VarTheta => Self::VarTheta,
505            E::VarKappa => Self::VarKappa,
506            E::VarPi => Self::VarPi,
507            E::VarRho => Self::VarRho,
508            E::VarSigma => Self::VarSigma,
509            E::VarPhi => Self::VarPhi,
510            E::UpperGamma => Self::UpperGamma,
511            E::UpperDelta => Self::UpperDelta,
512            E::UpperTheta => Self::UpperTheta,
513            E::UpperLambda => Self::UpperLambda,
514            E::UpperXi => Self::UpperXi,
515            E::UpperPi => Self::UpperPi,
516            E::UpperSigma => Self::UpperSigma,
517            E::UpperUpsilon => Self::UpperUpsilon,
518            E::UpperPhi => Self::UpperPhi,
519            E::UpperPsi => Self::UpperPsi,
520            E::UpperOmega => Self::UpperOmega,
521            E::TextBf => Self::TextBf,
522            E::TextIt => Self::TextIt,
523            E::TextSc => Self::TextSc,
524            E::TextTt => Self::TextTt,
525            E::Emph => Self::Emph,
526            E::Underline => Self::Underline,
527            E::Identifier => Self::Identifier,
528            E::StringLiteral => Self::StringLiteral,
529            E::Number => Self::Number,
530            E::Backslash => Self::Backslash,
531            E::LeftBrace => Self::LeftBrace,
532            E::RightBrace => Self::RightBrace,
533            E::LeftBracket => Self::LeftBracket,
534            E::RightBracket => Self::RightBracket,
535            E::LeftParen => Self::LeftParen,
536            E::RightParen => Self::RightParen,
537            E::Dollar => Self::Dollar,
538            E::DoubleDollar => Self::DoubleDollar,
539            E::Ampersand => Self::Ampersand,
540            E::Percent => Self::Percent,
541            E::Hash => Self::Hash,
542            E::Caret => Self::Caret,
543            E::Underscore => Self::Underscore,
544            E::Tilde => Self::Tilde,
545            E::Equal => Self::Equal,
546            E::Equals => Self::Equals,
547            E::Plus => Self::Plus,
548            E::Minus => Self::Minus,
549            E::Star => Self::Star,
550            E::Slash => Self::Slash,
551            E::Pipe => Self::Pipe,
552            E::Less => Self::Less,
553            E::LessThan => Self::LessThan,
554            E::Greater => Self::Greater,
555            E::GreaterThan => Self::GreaterThan,
556            E::Exclamation => Self::Exclamation,
557            E::Question => Self::Question,
558            E::At => Self::At,
559            E::Colon => Self::Colon,
560            E::Semicolon => Self::Semicolon,
561            E::Comma => Self::Comma,
562            E::Dot => Self::Dot,
563            E::Comment => Self::Comment,
564            E::Whitespace => Self::Whitespace,
565            E::Newline => Self::Newline,
566            E::BeginKeyword => Self::BeginKeyword,
567            E::EndKeyword => Self::EndKeyword,
568            E::DocumentclassKeyword => Self::DocumentclassKeyword,
569            E::UsepackageKeyword => Self::UsepackageKeyword,
570            E::SectionKeyword => Self::SectionKeyword,
571            E::SubsectionKeyword => Self::SubsectionKeyword,
572            E::SubsubsectionKeyword => Self::SubsubsectionKeyword,
573            E::ChapterKeyword => Self::ChapterKeyword,
574            E::PartKeyword => Self::PartKeyword,
575            E::TitleKeyword => Self::TitleKeyword,
576            E::AuthorKeyword => Self::AuthorKeyword,
577            E::DateKeyword => Self::DateKeyword,
578            E::MaketitleKeyword => Self::MaketitleKeyword,
579            E::TableofcontentsKeyword => Self::TableofcontentsKeyword,
580            E::ItemKeyword => Self::ItemKeyword,
581            E::LabelKeyword => Self::LabelKeyword,
582            E::RefKeyword => Self::RefKeyword,
583            E::CiteKeyword => Self::CiteKeyword,
584            E::IncludegraphicsKeyword => Self::IncludegraphicsKeyword,
585            E::TextbfKeyword => Self::TextbfKeyword,
586            E::TextitKeyword => Self::TextitKeyword,
587            E::EmphKeyword => Self::EmphKeyword,
588            E::Eof => Self::Eof,
589        }
590    }
591}
oak_tex/lexer/token_type.rs

oak_tex/lexer/
token_type.rs
