retex_engine/
token.rs

1//! Tokens and token lists.
2//!
3//! A token is a `u32`:
4//!
5//! * a *character token* packs a command code and a character:
6//!   `(cmd << 21) | chr`, with `chr ≤ 0x1F_FFFF`. The command is usually a
7//!   category code (≤ 15), but LuaTeX's `token.create(chr, cmd)` and
8//!   `token.new` make such tokens for any command (e.g. `char_given`);
9//! * a *control-sequence token* is `CS_TOKEN_FLAG + cs`, where `cs` is the
10//!   id of the control sequence in the hash table.
11//!
12//! Inside macro definitions TeX also uses `match` (13) and `end_match` (14)
13//! tokens in the parameter text and `out_param` (5) tokens in the body.
14
15use std::rc::Rc;
16
17use crate::cmd::Cmd;
18use crate::hash::CsId;
19
20/// The offset of control-sequence tokens.
21pub const CS_TOKEN_FLAG: u32 = 0x2000_0000;
22/// The multiplier of the command code in character tokens.
23const CMD_SHIFT: u32 = 21;
24const CHR_MASK: u32 = (1 << CMD_SHIFT) - 1;
25
26/// A TeX token.
27#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug, PartialOrd, Ord, Default)]
28pub struct Token(pub u32);
29
30impl Token {
31    /// A character token with command `cmd` (normally a category code).
32    #[inline]
33    pub const fn char(cmd: Cmd, chr: u32) -> Token {
34        debug_assert!(((cmd as u32) << CMD_SHIFT) < CS_TOKEN_FLAG);
35        Token(((cmd as u32) << CMD_SHIFT) | chr)
36    }
37
38    /// A control-sequence token.
39    #[inline]
40    pub const fn cs(cs: CsId) -> Token {
41        Token(CS_TOKEN_FLAG + cs.0)
42    }
43
44    /// Whether this is a control-sequence token.
45    #[inline]
46    pub const fn is_cs(self) -> bool {
47        self.0 >= CS_TOKEN_FLAG
48    }
49
50    /// The control sequence of a control-sequence token.
51    #[inline]
52    pub const fn cs_id(self) -> CsId {
53        CsId(self.0 - CS_TOKEN_FLAG)
54    }
55
56    /// The command (category) code of a character token.
57    #[inline]
58    pub fn cmd(self) -> Cmd {
59        debug_assert!(!self.is_cs());
60        Cmd::ALL[(self.0 >> CMD_SHIFT) as usize]
61    }
62
63    /// The raw command code of a character token (no enum conversion).
64    #[inline]
65    pub const fn cmd_code(self) -> u32 {
66        self.0 >> CMD_SHIFT
67    }
68
69    /// The character of a character token.
70    #[inline]
71    pub const fn chr(self) -> u32 {
72        self.0 & CHR_MASK
73    }
74
75    /// A `match` token for parameter `n` in a parameter text. TeX stores the
76    /// parameter character (e.g. `#`) here.
77    pub const fn match_token(chr: u32) -> Token {
78        Token::char(Cmd::ParEnd, chr)
79    }
80
81    /// The `end_match` token separating parameter text from the body.
82    pub const END_MATCH: Token = Token::char(Cmd::Stop, 0);
83
84    /// An `out_param` token for parameter `n` (1–9) in a macro body.
85    pub const fn out_param(n: u32) -> Token {
86        Token::char(Cmd::CarRet, n)
87    }
88
89    /// A left brace `{` token (category 1, character `{`).
90    pub const LEFT_BRACE: Token = Token::char(Cmd::LeftBrace, '{' as u32);
91    /// A right brace `}` token.
92    pub const RIGHT_BRACE: Token = Token::char(Cmd::RightBrace, '}' as u32);
93    /// A space token (category 10, character 32).
94    pub const SPACE: Token = Token::char(Cmd::Spacer, ' ' as u32);
95    /// An `other` character token.
96    pub const fn other(c: u32) -> Token {
97        Token::char(Cmd::OtherChar, c)
98    }
99    /// A `letter` character token.
100    pub const fn letter(c: u32) -> Token {
101        Token::char(Cmd::Letter, c)
102    }
103}
104
105/// An immutable, shared token list (a macro body, the contents of a token
106/// register, …).
107pub type TokenList = Rc<[Token]>;
108
109/// The empty token list.
110pub fn empty_list() -> TokenList {
111    thread_local! {
112        static EMPTY: TokenList = Rc::from(Vec::new());
113    }
114    EMPTY.with(Rc::clone)
115}
116
117/// A macro definition.
118#[derive(Debug, Clone)]
119pub struct Macro {
120    /// e-TeX `\protected` flag.
121    pub protected: bool,
122    /// Parameter text (match tokens), the `END_MATCH` token, then the body.
123    pub tokens: TokenList,
124    /// The native implementation of this definition, found on first call
125    /// ([`crate::native`]): 0 = not looked up yet, 1 = none, else the
126    /// registry index + 2.
127    pub native: std::cell::Cell<u16>,
128    /// Where the definition was made, when origins are recorded
129    /// ([`crate::profile::Origins`]); 0 if unknown. Set when a format's
130    /// origins are loaded, for definitions shared by several names.
131    pub origin: std::cell::Cell<u32>,
132}
133
134impl Macro {
135    /// A macro with parameter text and body `tokens`.
136    pub fn new(protected: bool, tokens: TokenList) -> Macro {
137        Macro {
138            protected,
139            tokens,
140            native: std::cell::Cell::new(0),
141            origin: std::cell::Cell::new(0),
142        }
143    }
144}
145
146// Equality (`\ifx`) is by definition only, not by the native lookup cache.
147impl PartialEq for Macro {
148    fn eq(&self, other: &Macro) -> bool {
149        self.protected == other.protected && self.tokens == other.tokens
150    }
151}
152
153impl Eq for Macro {}
154
155#[cfg(test)]
156mod tests {
157    use super::*;
158
159    #[test]
160    fn macro_origin_fits_in_padding() {
161        assert_eq!(std::mem::size_of::<Macro>(), 24);
162    }
163
164    #[test]
165    fn packing() {
166        let t = Token::char(Cmd::Letter, 0x10FFFF);
167        assert!(!t.is_cs());
168        assert_eq!(t.cmd(), Cmd::Letter);
169        assert_eq!(t.chr(), 0x10FFFF);
170        let c = Token::cs(CsId(12345));
171        assert!(c.is_cs());
172        assert_eq!(c.cs_id(), CsId(12345));
173        assert!(Token::char(Cmd::DelimNum, 0x1F_FFFF).0 < CS_TOKEN_FLAG);
174    }
175}
retex_engine/token.rs

retex_engine/
token.rs
