1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
//! TeX's category codes — the sixteen classes every character belongs to.
//!
//! `tex.web` §207 fixes the numbering, and the classification is what makes TeX
//! reconfigurable at the character level: `\catcode`\@=11` is how plain.tex makes
//! `@` a letter for the duration of a macro file. Nothing here decides what a
//! character MEANS; that is the mouth's job (`crate::lexer`). This only says
//! which of the sixteen classes it is in right now.
/// A category code, numbered as `tex.web` numbers them.
#[derive(Clone, Copy, PartialEq, Eq, Debug)]
#[repr(u8)]
pub enum Cat {
Escape = 0,
BeginGroup = 1,
EndGroup = 2,
MathShift = 3,
AlignTab = 4,
EndLine = 5,
Param = 6,
Superscript = 7,
Subscript = 8,
Ignored = 9,
Space = 10,
Letter = 11,
Other = 12,
Active = 13,
Comment = 14,
Invalid = 15,
}
impl Cat {
/// Every category, in `tex.web`'s numbering order.
pub const ALL: [Cat; 16] = [
Cat::Escape,
Cat::BeginGroup,
Cat::EndGroup,
Cat::MathShift,
Cat::AlignTab,
Cat::EndLine,
Cat::Param,
Cat::Superscript,
Cat::Subscript,
Cat::Ignored,
Cat::Space,
Cat::Letter,
Cat::Other,
Cat::Active,
Cat::Comment,
Cat::Invalid,
];
/// The name `tex.web` gives this category, for a reference table or a
/// diagnostic. Not the Rust variant name: `tex.web` writes "end of line",
/// not "EndLine".
pub fn name(self) -> &'static str {
match self {
Cat::Escape => "escape",
Cat::BeginGroup => "begin group",
Cat::EndGroup => "end group",
Cat::MathShift => "math shift",
Cat::AlignTab => "alignment tab",
Cat::EndLine => "end of line",
Cat::Param => "parameter",
Cat::Superscript => "superscript",
Cat::Subscript => "subscript",
Cat::Ignored => "ignored",
Cat::Space => "space",
Cat::Letter => "letter",
Cat::Other => "other",
Cat::Active => "active",
Cat::Comment => "comment",
Cat::Invalid => "invalid",
}
}
}
/// The catcode of every character, mutable exactly as `\catcode` makes it.
///
/// INITEX's defaults (`tex.web` §232) are deliberately sparse: only `\`, `%`,
/// the letters, the space, the carriage return and the null/delete characters
/// are special. `{`, `}`, `$`, `&`, `#`, `^`, `_` and `~` get their familiar
/// meanings from plain.tex, NOT from the engine — which is why a bare INITEX
/// run treats `{` as an ordinary character. Reproducing that split matters: a
/// document that sets its own catcodes has to see the same starting point.
#[derive(Clone)]
pub struct CatTable {
codes: [Cat; 256],
/// The codes of characters past 255 that have been assigned one. LuaTeX
/// reads characters, not bytes, and every one of them has a catcode
/// (`\catcode"0410=11`), `other` until assigned -- so this is sparse.
wide: rustc_hash::FxHashMap<u32, Cat>,
/// Bumped by every `set`.
///
/// Lexing ahead of the expander is only valid while the table it was done
/// under still holds. A counter is what lets a cached token stream say "I
/// was produced under generation N" and be thrown away the moment
/// `\catcode` moves the table on, without comparing 256 bytes each time.
generation: u32,
/// `\endlinechar` (`tex.web` §240: 13 in INITEX), which the mouth appends to
/// every line it reads (§362) -- nothing when it is outside 0..=255. It is
/// a §236 integer parameter, kept here as well because the table is what
/// the mouth is handed; `crate::expand` keeps the two in step.
end_line_char: i64,
}
impl Default for CatTable {
fn default() -> Self {
Self::new()
}
}
impl CatTable {
/// INITEX's table, before any format or macro file has run.
pub fn new() -> Self {
let mut codes = [Cat::Other; 256];
for c in b'A'..=b'Z' {
codes[c as usize] = Cat::Letter;
}
for c in b'a'..=b'z' {
codes[c as usize] = Cat::Letter;
}
codes[b'\\' as usize] = Cat::Escape;
codes[b'%' as usize] = Cat::Comment;
// §232: `^^M` (the carriage return) is the only end-of-line character.
// A physical line end in the file is not a character at all -- the
// mouth appends `\endlinechar` in its place -- so `^^J` is `Other`.
codes[b'\r' as usize] = Cat::EndLine;
codes[b' ' as usize] = Cat::Space;
codes[0] = Cat::Ignored;
codes[127] = Cat::Invalid;
Self {
codes,
wide: rustc_hash::FxHashMap::default(),
generation: 0,
end_line_char: 13,
}
}
/// Which revision of the table this is; see [`CatTable::generation`]'s field.
pub fn generation(&self) -> u32 {
self.generation
}
/// `\endlinechar`, as a character: `None` when it is outside 0..=255, which
/// is how `tex.web` §362 says "append nothing".
pub fn end_line_char(&self) -> Option<char> {
u8::try_from(self.end_line_char).ok().map(char::from)
}
/// Record a new `\endlinechar`. It changes what the mouth produces, so
/// anything lexed ahead under the old value is stale, as after `\catcode`.
pub fn set_end_line_char(&mut self, v: i64) {
self.end_line_char = v;
self.generation = self.generation.wrapping_add(1);
}
/// The table textcodes.c's `set_cat_code` makes for a table nothing has
/// made yet: every code `CATCODEDEFAULT`, which is 12.
pub fn all_other() -> Self {
let mut t = Self::new();
t.codes = [Cat::Other; 256];
t
}
/// Take a generation past `other`'s: a table switched in by
/// `\catcodetable` must not look like one the mouth already read ahead
/// under, whatever its own count says.
pub fn follow_generation(&mut self, other: u32) {
self.generation = self.generation.max(other).wrapping_add(1);
}
pub fn get(&self, c: char) -> Cat {
match u32::from(c) {
// Outside Latin-1, `Other`. TeX82 reads BYTES, so a character up
// there is a run of `Other` bytes to it and never part of a control
// word. Calling them Letters here made them part of one: in a UTF-8
// document `\textgreater→key` lexed as a single control sequence
// named `textgreater→key`, so a real document full of arrows and
// dashes failed on names that do not exist. `Other` is both closer
// to what tex does and the only choice that ends a control word
// where tex ends it. A LuaTeX run can give one a code of its own
// (`\catcode"0410=11`), and then it has that.
n if n > 255 => self.wide.get(&n).copied().unwrap_or(Cat::Other),
n => self.codes[n as usize],
}
}
pub fn set(&mut self, c: char, cat: Cat) {
if let Ok(i) = usize::try_from(u32::from(c)) {
if i < 256 {
// Bump even when the value is unchanged: `\catcode` reassigning
// a character its current class is still a point where anything
// lexed ahead has to be re-checked, and pretending otherwise
// would make correctness depend on the old value.
self.codes[i] = cat;
} else {
self.wide.insert(u32::from(c), cat);
}
self.generation = self.generation.wrapping_add(1);
}
}
}
/// `tex.web`'s numbering, for `\catcode` reads and writes.
pub fn cat_from_i64(n: i64) -> Option<Cat> {
Some(match n {
0 => Cat::Escape,
1 => Cat::BeginGroup,
2 => Cat::EndGroup,
3 => Cat::MathShift,
4 => Cat::AlignTab,
5 => Cat::EndLine,
6 => Cat::Param,
7 => Cat::Superscript,
8 => Cat::Subscript,
9 => Cat::Ignored,
10 => Cat::Space,
11 => Cat::Letter,
12 => Cat::Other,
13 => Cat::Active,
14 => Cat::Comment,
15 => Cat::Invalid,
_ => return None,
})
}