1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
//! Tokens and token lists.
//!
//! A token is a `u32`:
//!
//! * a *character token* packs a command code and a character:
//! `(cmd << 21) | chr`, with `chr ≤ 0x1F_FFFF`. The command is usually a
//! category code (≤ 15), but LuaTeX's `token.create(chr, cmd)` and
//! `token.new` make such tokens for any command (e.g. `char_given`);
//! * a *control-sequence token* is `CS_TOKEN_FLAG + cs`, where `cs` is the
//! id of the control sequence in the hash table.
//!
//! Inside macro definitions TeX also uses `match` (13) and `end_match` (14)
//! tokens in the parameter text and `out_param` (5) tokens in the body.
use std::rc::Rc;
use crate::cmd::Cmd;
use crate::hash::CsId;
/// The offset of control-sequence tokens.
pub const CS_TOKEN_FLAG: u32 = 0x2000_0000;
/// The multiplier of the command code in character tokens.
const CMD_SHIFT: u32 = 21;
const CHR_MASK: u32 = (1 << CMD_SHIFT) - 1;
/// A TeX token.
#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug, PartialOrd, Ord, Default)]
pub struct Token(pub u32);
impl Token {
/// A character token with command `cmd` (normally a category code).
#[inline]
pub const fn char(cmd: Cmd, chr: u32) -> Token {
debug_assert!(((cmd as u32) << CMD_SHIFT) < CS_TOKEN_FLAG);
Token(((cmd as u32) << CMD_SHIFT) | chr)
}
/// A control-sequence token.
#[inline]
pub const fn cs(cs: CsId) -> Token {
Token(CS_TOKEN_FLAG + cs.0)
}
/// Whether this is a control-sequence token.
#[inline]
pub const fn is_cs(self) -> bool {
self.0 >= CS_TOKEN_FLAG
}
/// The control sequence of a control-sequence token.
#[inline]
pub const fn cs_id(self) -> CsId {
CsId(self.0 - CS_TOKEN_FLAG)
}
/// The command (category) code of a character token.
#[inline]
pub fn cmd(self) -> Cmd {
debug_assert!(!self.is_cs());
Cmd::ALL[(self.0 >> CMD_SHIFT) as usize]
}
/// The raw command code of a character token (no enum conversion).
#[inline]
pub const fn cmd_code(self) -> u32 {
self.0 >> CMD_SHIFT
}
/// The character of a character token.
#[inline]
pub const fn chr(self) -> u32 {
self.0 & CHR_MASK
}
/// A `match` token for parameter `n` in a parameter text. TeX stores the
/// parameter character (e.g. `#`) here.
pub const fn match_token(chr: u32) -> Token {
Token::char(Cmd::ParEnd, chr)
}
/// The `end_match` token separating parameter text from the body.
pub const END_MATCH: Token = Token::char(Cmd::Stop, 0);
/// An `out_param` token for parameter `n` (1–9) in a macro body.
pub const fn out_param(n: u32) -> Token {
Token::char(Cmd::CarRet, n)
}
/// A left brace `{` token (category 1, character `{`).
pub const LEFT_BRACE: Token = Token::char(Cmd::LeftBrace, '{' as u32);
/// A right brace `}` token.
pub const RIGHT_BRACE: Token = Token::char(Cmd::RightBrace, '}' as u32);
/// A space token (category 10, character 32).
pub const SPACE: Token = Token::char(Cmd::Spacer, ' ' as u32);
/// An `other` character token.
pub const fn other(c: u32) -> Token {
Token::char(Cmd::OtherChar, c)
}
/// A `letter` character token.
pub const fn letter(c: u32) -> Token {
Token::char(Cmd::Letter, c)
}
}
/// An immutable, shared token list (a macro body, the contents of a token
/// register, …).
pub type TokenList = Rc<[Token]>;
/// The empty token list.
pub fn empty_list() -> TokenList {
thread_local! {
static EMPTY: TokenList = Rc::from(Vec::new());
}
EMPTY.with(Rc::clone)
}
/// A macro definition.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Macro {
/// e-TeX `\protected` flag.
pub protected: bool,
/// Parameter text (match tokens), the `END_MATCH` token, then the body.
pub tokens: TokenList,
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn packing() {
let t = Token::char(Cmd::Letter, 0x10FFFF);
assert!(!t.is_cs());
assert_eq!(t.cmd(), Cmd::Letter);
assert_eq!(t.chr(), 0x10FFFF);
let c = Token::cs(CsId(12345));
assert!(c.is_cs());
assert_eq!(c.cs_id(), CsId(12345));
assert!(Token::char(Cmd::DelimNum, 0x1F_FFFF).0 < CS_TOKEN_FLAG);
}
}