katex_parser/
token.rs

1use std::sync::Arc;
2
3use crate::source_location::SourceLocation;
4
5#[derive(Debug, Clone, PartialEq)]
6/// A lexed or macro-expanded token.
7pub struct Token {
8    pub text: String,
9    pub loc: Option<SourceLocation>,
10    pub noexpand: bool,
11    pub treat_as_relax: bool,
12}
13
14impl Token {
15    pub fn new(text: impl Into<String>, loc: Option<SourceLocation>) -> Self {
16        Token {
17            text: text.into(),
18            loc,
19            noexpand: false,
20            treat_as_relax: false,
21        }
22    }
23
24    pub fn eof(input: impl Into<Arc<str>>, offset: usize) -> Self {
25        let input = input.into();
26        Token::new(
27            "EOF",
28            Some(SourceLocation::new(input, offset, offset)),
29        )
30    }
31
32    pub fn range(&self, end_token: &Token, text: impl Into<String>) -> Self {
33        if let (Some(start_loc), Some(end_loc)) = (&self.loc, &end_token.loc) {
34            Token::new(text, Some(SourceLocation::range(start_loc, end_loc)))
35        } else {
36            Token::new(text, None)
37        }
38    }
39}
40
41/// Returns the source location of a token, if any.
42pub fn token_location(token: Option<&Token>) -> Option<SourceLocation> {
43    token.and_then(|token| token.loc.clone())
44}
katex_parser/token.rs

katex_parser/
token.rs
